[2024-12-16 17:49:25.101496] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8F-000629642BDC7ACF-0-0] [lt=22] [DDL] execute ddl like stmt(ret=0, cost=18, ddl_arg=NULL) [2024-12-16 17:49:25.150061] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F7F6-0-0] [lt=41] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202081794000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202110029000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:0, operation_type_:0, create_scn:{val:1734359202119679000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:25.152665] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [128655][T1_StandbySchem][T1][YB42C0A89F89-00062963DD93FAC3-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:25.153814] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F7F6-0-0] [lt=159] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-16 17:49:25.153835] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F7F6-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:25.194707] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [130219][T1002_FrzInfoDe][T1002][YB42C0A89F89-00062963EAD36921-0-0] [lt=41] get number(ret=0, service_type_=0, range=1, base_id=1734400165190130000, start_id=1734400165190130000, end_id=1734400165190130001) [2024-12-16 17:49:25.203427] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [130219][T1002_FrzInfoDe][T1002][YB42C0A89F89-00062963EAD36921-0-0] [lt=38] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1734400165190130000, v:0}, tenant_id=1002) [2024-12-16 17:49:25.214909] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128853][T1_L0_G0][T1][YB42C0A89F8F-000629642D9C5714-0-0] [lt=49] [DDL] execute ddl like stmt(ret=0, cost=13, ddl_arg=NULL) [2024-12-16 17:49:25.290964] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8A-00062963DFD99B24-0-0] [lt=216] [DDL] execute ddl like stmt(ret=0, cost=102, ddl_arg=NULL) [2024-12-16 17:49:25.293973] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F40C-0-0] [lt=19] [COMMON_LS_SERVICE] start to create ls(info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, create_scn={val:1734359202119679000, v:0}) [2024-12-16 17:49:25.308843] WDIAG [RS] do_create_user_ls (ob_common_ls_service.cpp:269) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F40C-0-0] [lt=9][errcode=-4672] failed to create user ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, locality_array=[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], create_scn={val:1734359202119679000, v:0}, palf_base_info={prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}, curr_lsn:{lsn:18446744073709551615}}, create_with_palf=false, source_tenant_id=0) [2024-12-16 17:49:25.308890] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:274) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F40C-0-0] [lt=36] [COMMON_LS_SERVICE] end to create ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, cost=14786) [2024-12-16 17:49:25.308903] WDIAG [RS] try_create_ls_ (ob_common_ls_service.cpp:179) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F40C-0-0] [lt=12][errcode=-4672] failed to create new ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", status_info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, recovery_stat={tenant_id:1002, ls_id:{id:1003}, sync_scn:{val:1, v:0}, readable_scn:{val:1, v:0}, create_scn:{val:1734359202119679000, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:9223372036854775807, config_seq:-1}}) [2024-12-16 17:49:25.308923] WDIAG [RS] do_work (ob_common_ls_service.cpp:117) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F40C-0-0] [lt=18][errcode=0] failed to create ls(ret=0, ret="OB_SUCCESS", tmp_ret=-4672, tmp_ret="OB_REPLICA_NUM_NOT_ENOUGH", user_tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:25.314875] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F40C-0-0] [lt=219] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}, {unit_group_id:1002, unit_status:0, ls_group_ids:[]}]) [2024-12-16 17:49:25.318076] INFO [RS] do_work (ob_common_ls_service.cpp:138) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F40C-0-0] [lt=44] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:25.352123] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8D-000629641B98C905-0-0] [lt=81] [DDL] execute ddl like stmt(ret=0, cost=23, ddl_arg=NULL) [2024-12-16 17:49:25.511146] INFO [RS] remove_inactive_ddl_task (ob_ddl_scheduler.cpp:2588) [128994][HeartBeatCheckT][T0][Y0-0000000000000000-0-0] [lt=10] need remove task(remove_task_ids=[]) [2024-12-16 17:49:25.511193] INFO [RS] remove_inactive_ddl_task (ob_ddl_scheduler.cpp:2607) [128994][HeartBeatCheckT][T0][Y0-0000000000000000-0-0] [lt=43] remove all timeout ddl task succeed [2024-12-16 17:49:25.531403] INFO [RS] do_work (ob_common_ls_service.cpp:138) [128656][T1_COMMONLSSe][T1][YB42C0A89F89-00062963E043FA74-0-0] [lt=42] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:25.679548] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [129880][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=45] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1734400164727340, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1734400162429794003, replayable_scn:1734400162429794003, readable_scn:1734359204309426002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-16 17:49:25.692374] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [128684][T1_HBService][T1][YB42C0A89F89-00062963E003A9BE-0-0] [lt=44] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:25.736410] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.159.137:2882") [2024-12-16 17:49:25.736454] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=43] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="192.168.159.138:2882") [2024-12-16 17:49:25.736466] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="192.168.159.139:2882") [2024-12-16 17:49:25.736558] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=91] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=3, dest="192.168.159.141:2882") [2024-12-16 17:49:25.736565] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=4, dest="192.168.159.142:2882") [2024-12-16 17:49:25.736571] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=5, dest="192.168.159.143:2882") [2024-12-16 17:49:25.736690] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=109] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=4733, return_code_array=[0, 0, 0, 0, 0, 0], broadcast_times=2) [2024-12-16 17:49:25.761993] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129974][T1001_StandbySc][T1001][YB42C0A89F89-00062963E5D3F94A-0-0] [lt=25] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:25.894074] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8B-00062963DFDFA01A-0-0] [lt=39] [DDL] execute ddl like stmt(ret=0, cost=17, ddl_arg=NULL) [2024-12-16 17:49:25.895204] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F41C-0-0] [lt=61] is primary tenant, no need check readable_scn [2024-12-16 17:49:25.917433] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F41C-0-0] [lt=43] is primary tenant, no need check readable_scn [2024-12-16 17:49:25.928367] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F41C-0-0] [lt=30] is primary tenant, no need check readable_scn [2024-12-16 17:49:26.016231] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8E-000629642010734F-0-0] [lt=23] [DDL] execute ddl like stmt(ret=0, cost=20, ddl_arg=NULL) [2024-12-16 17:49:26.020464] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129809][T1002_StandbySc][T1002][YB42C0A89F89-00062963E533F94C-0-0] [lt=73] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:26.152890] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [128655][T1_StandbySchem][T1][YB42C0A89F89-00062963DD93FAC4-0-0] [lt=36] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:26.157772] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F7F7-0-0] [lt=41] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202081794000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202110029000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:0, operation_type_:0, create_scn:{val:1734359202119679000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:26.160882] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F7F7-0-0] [lt=169] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-16 17:49:26.160901] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F7F7-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:26.321472] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F40D-0-0] [lt=39] [COMMON_LS_SERVICE] start to create ls(info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, create_scn={val:1734359202119679000, v:0}) [2024-12-16 17:49:26.325218] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128861][T1_L0_G0][T1][YB42C0A89F8A-00062963E0A9757C-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=55, ddl_arg=NULL) [2024-12-16 17:49:26.342209] WDIAG [RS] do_create_user_ls (ob_common_ls_service.cpp:269) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F40D-0-0] [lt=7][errcode=-4672] failed to create user ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, locality_array=[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], create_scn={val:1734359202119679000, v:0}, palf_base_info={prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}, curr_lsn:{lsn:18446744073709551615}}, create_with_palf=false, source_tenant_id=0) [2024-12-16 17:49:26.342277] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:274) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F40D-0-0] [lt=57] [COMMON_LS_SERVICE] end to create ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, cost=20748) [2024-12-16 17:49:26.342295] WDIAG [RS] try_create_ls_ (ob_common_ls_service.cpp:179) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F40D-0-0] [lt=16][errcode=-4672] failed to create new ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", status_info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, recovery_stat={tenant_id:1002, ls_id:{id:1003}, sync_scn:{val:1, v:0}, readable_scn:{val:1, v:0}, create_scn:{val:1734359202119679000, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:9223372036854775807, config_seq:-1}}) [2024-12-16 17:49:26.342316] WDIAG [RS] do_work (ob_common_ls_service.cpp:117) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F40D-0-0] [lt=18][errcode=0] failed to create ls(ret=0, ret="OB_SUCCESS", tmp_ret=-4672, tmp_ret="OB_REPLICA_NUM_NOT_ENOUGH", user_tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:26.347420] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F40D-0-0] [lt=42] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}, {unit_group_id:1002, unit_status:0, ls_group_ids:[]}]) [2024-12-16 17:49:26.350711] INFO [RS] do_work (ob_common_ls_service.cpp:138) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F40D-0-0] [lt=48] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:26.533885] INFO [RS] do_work (ob_common_ls_service.cpp:138) [128656][T1_COMMONLSSe][T1][YB42C0A89F89-00062963E043FA75-0-0] [lt=53] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:26.732691] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=14] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1734400164433716003, replayable_scn:1734400164433716003, readable_scn:1734359204309426002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1734400166727617, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1734400164433716003, replayable_scn:1734400164433716003, readable_scn:1734359204309426002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1734400166727617, ora_rowscn_=1734400165889685001, content_changed=true) [2024-12-16 17:49:26.736307] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=26] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.159.137:2882") [2024-12-16 17:49:26.736333] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=25] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="192.168.159.138:2882") [2024-12-16 17:49:26.736342] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="192.168.159.139:2882") [2024-12-16 17:49:26.736349] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=3, dest="192.168.159.141:2882") [2024-12-16 17:49:26.736356] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=4, dest="192.168.159.142:2882") [2024-12-16 17:49:26.736362] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=5, dest="192.168.159.143:2882") [2024-12-16 17:49:26.737816] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-16 17:49:26.762479] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129974][T1001_StandbySc][T1001][YB42C0A89F89-00062963E5D3F94B-0-0] [lt=25] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:26.942040] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F41D-0-0] [lt=20] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.159.137:2882", readable_scn:{val:1734400166899057000, v:0}}, {server:"192.168.159.138:2882", readable_scn:{val:1734400166899057000, v:0}}, {server:"192.168.159.139:2882", readable_scn:{val:1734400166899057000, v:0}}], last_dump_ts_=1734400161742752, force_dump=false) [2024-12-16 17:49:26.942156] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F41D-0-0] [lt=107] is primary tenant, no need check readable_scn [2024-12-16 17:49:26.970749] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F41D-0-0] [lt=97] is primary tenant, no need check readable_scn [2024-12-16 17:49:26.979212] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F41D-0-0] [lt=21] is primary tenant, no need check readable_scn [2024-12-16 17:49:27.020590] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129809][T1002_StandbySc][T1002][YB42C0A89F89-00062963E533F94D-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:27.022133] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [128685][T1_HBService][T1][YB42C0A89F89-00062963E023AA0B-0-0] [lt=45] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:27.023907] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F89-00062963E103A9E1-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=17, ddl_arg=NULL) [2024-12-16 17:49:27.109855] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8F-000629642BDC7AD0-0-0] [lt=166] [DDL] execute ddl like stmt(ret=0, cost=123, ddl_arg=NULL) [2024-12-16 17:49:27.153050] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [128655][T1_StandbySchem][T1][YB42C0A89F89-00062963DD93FAC5-0-0] [lt=79] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:27.165582] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F7F8-0-0] [lt=39] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202081794000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202110029000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:0, operation_type_:0, create_scn:{val:1734359202119679000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:27.170952] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F7F8-0-0] [lt=185] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-16 17:49:27.171004] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F7F8-0-0] [lt=50] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:27.305233] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8A-00062963DFD99B25-0-0] [lt=29] [DDL] execute ddl like stmt(ret=0, cost=60, ddl_arg=NULL) [2024-12-16 17:49:27.355247] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F40E-0-0] [lt=38] [COMMON_LS_SERVICE] start to create ls(info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, create_scn={val:1734359202119679000, v:0}) [2024-12-16 17:49:27.362361] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8D-000629641B98C906-0-0] [lt=57] [DDL] execute ddl like stmt(ret=0, cost=44, ddl_arg=NULL) [2024-12-16 17:49:27.374269] WDIAG [RS] do_create_user_ls (ob_common_ls_service.cpp:269) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F40E-0-0] [lt=7][errcode=-4672] failed to create user ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, locality_array=[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], create_scn={val:1734359202119679000, v:0}, palf_base_info={prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}, curr_lsn:{lsn:18446744073709551615}}, create_with_palf=false, source_tenant_id=0) [2024-12-16 17:49:27.374319] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:274) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F40E-0-0] [lt=38] [COMMON_LS_SERVICE] end to create ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, cost=19031) [2024-12-16 17:49:27.374410] WDIAG [RS] try_create_ls_ (ob_common_ls_service.cpp:179) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F40E-0-0] [lt=89][errcode=-4672] failed to create new ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", status_info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, recovery_stat={tenant_id:1002, ls_id:{id:1003}, sync_scn:{val:1, v:0}, readable_scn:{val:1, v:0}, create_scn:{val:1734359202119679000, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:9223372036854775807, config_seq:-1}}) [2024-12-16 17:49:27.374441] WDIAG [RS] do_work (ob_common_ls_service.cpp:117) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F40E-0-0] [lt=27][errcode=0] failed to create ls(ret=0, ret="OB_SUCCESS", tmp_ret=-4672, tmp_ret="OB_REPLICA_NUM_NOT_ENOUGH", user_tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:27.380180] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F40E-0-0] [lt=107] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}, {unit_group_id:1002, unit_status:0, ls_group_ids:[]}]) [2024-12-16 17:49:27.383711] INFO [RS] do_work (ob_common_ls_service.cpp:138) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F40E-0-0] [lt=48] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:27.536245] INFO [RS] do_work (ob_common_ls_service.cpp:138) [128656][T1_COMMONLSSe][T1][YB42C0A89F89-00062963E043FA76-0-0] [lt=83] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:27.701251] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [128684][T1_HBService][T1][YB42C0A89F89-00062963E003A9BF-0-0] [lt=23] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:27.734178] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.159.137:2882") [2024-12-16 17:49:27.734538] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=358] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="192.168.159.138:2882") [2024-12-16 17:49:27.734551] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="192.168.159.139:2882") [2024-12-16 17:49:27.734560] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=3, dest="192.168.159.141:2882") [2024-12-16 17:49:27.734645] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=85] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=4, dest="192.168.159.142:2882") [2024-12-16 17:49:27.734654] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=5, dest="192.168.159.143:2882") [2024-12-16 17:49:27.769327] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129974][T1001_StandbySc][T1001][YB42C0A89F89-00062963E5D3F94C-0-0] [lt=23] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:27.897351] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8B-00062963DFDFA01B-0-0] [lt=77] [DDL] execute ddl like stmt(ret=0, cost=19, ddl_arg=NULL) [2024-12-16 17:49:28.002405] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F41E-0-0] [lt=18] is primary tenant, no need check readable_scn [2024-12-16 17:49:28.023233] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129809][T1002_StandbySc][T1002][YB42C0A89F89-00062963E533F94E-0-0] [lt=76] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:28.023411] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8E-0006296420107350-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=20, ddl_arg=NULL) [2024-12-16 17:49:28.027012] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F41E-0-0] [lt=32] is primary tenant, no need check readable_scn [2024-12-16 17:49:28.040376] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F41E-0-0] [lt=21] is primary tenant, no need check readable_scn [2024-12-16 17:49:28.153230] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [128655][T1_StandbySchem][T1][YB42C0A89F89-00062963DD93FAC6-0-0] [lt=96] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:28.174873] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F7F9-0-0] [lt=114] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202081794000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202110029000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:0, operation_type_:0, create_scn:{val:1734359202119679000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:28.177812] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F7F9-0-0] [lt=135] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-16 17:49:28.177943] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F7F9-0-0] [lt=128] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:28.197802] INFO [RS] finish (ob_mview_maintenance_task.cpp:282) [129841][T1002_TntShared][T1002][Y0-0000000000000000-0-0] [lt=33] mview maintenace task finish(tenant_id_=1002, round_=513, status_=100, error_code_=0, cost_us_=20001844, prepare_cost_us_=1, gc_mview_cost_us_=1345, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2024-12-16 17:49:28.252186] INFO [RS] get_dbms_sched_job_infos_in_tenant (ob_dbms_sched_table_operator.cpp:677) [129906][T1002_DBMSSched][T1002][Y0-0000000000000000-0-0] [lt=100] failed to get result(ret=-4008) [2024-12-16 17:49:28.388607] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F40F-0-0] [lt=40] [COMMON_LS_SERVICE] start to create ls(info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, create_scn={val:1734359202119679000, v:0}) [2024-12-16 17:49:28.414561] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [128532][DBMS_JOB_MASTER][T0][YB42C0A89F89-00062963DE235926-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2024-12-16 17:49:28.416386] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [128532][DBMS_JOB_MASTER][T0][YB42C0A89F89-00062963DE235926-0-0] [lt=130] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1734400168414131, delay:5000000, job_id:0, tenant_id:1002}, job_key=0x2b45b422a0e0, job_infos=[]) [2024-12-16 17:49:28.416453] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [128532][DBMS_JOB_MASTER][T0][YB42C0A89F89-00062963DE235926-0-0] [lt=62] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1734400173416449, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-16 17:49:28.421529] WDIAG [RS] do_create_user_ls (ob_common_ls_service.cpp:269) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F40F-0-0] [lt=8][errcode=-4672] failed to create user ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, locality_array=[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], create_scn={val:1734359202119679000, v:0}, palf_base_info={prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}, curr_lsn:{lsn:18446744073709551615}}, create_with_palf=false, source_tenant_id=0) [2024-12-16 17:49:28.421576] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:274) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F40F-0-0] [lt=37] [COMMON_LS_SERVICE] end to create ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, cost=32572) [2024-12-16 17:49:28.421589] WDIAG [RS] try_create_ls_ (ob_common_ls_service.cpp:179) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F40F-0-0] [lt=12][errcode=-4672] failed to create new ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", status_info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, recovery_stat={tenant_id:1002, ls_id:{id:1003}, sync_scn:{val:1, v:0}, readable_scn:{val:1, v:0}, create_scn:{val:1734359202119679000, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:9223372036854775807, config_seq:-1}}) [2024-12-16 17:49:28.421609] WDIAG [RS] do_work (ob_common_ls_service.cpp:117) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F40F-0-0] [lt=17][errcode=0] failed to create ls(ret=0, ret="OB_SUCCESS", tmp_ret=-4672, tmp_ret="OB_REPLICA_NUM_NOT_ENOUGH", user_tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:28.426464] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F40F-0-0] [lt=103] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}, {unit_group_id:1002, unit_status:0, ls_group_ids:[]}]) [2024-12-16 17:49:28.430067] INFO [RS] do_work (ob_common_ls_service.cpp:138) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F40F-0-0] [lt=43] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:28.528498] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [128532][DBMS_JOB_MASTER][T0][YB42C0A89F89-00062963DE235926-0-0] [lt=22] dbms job master wait timeout, no entry(ret=-4018) [2024-12-16 17:49:28.530696] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [128532][DBMS_JOB_MASTER][T0][YB42C0A89F89-00062963DE235926-0-0] [lt=23] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1734400168528195, delay:5000000, job_id:0, tenant_id:1001}, job_key=0x2b45b422a0a0, job_infos=[]) [2024-12-16 17:49:28.530778] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [128532][DBMS_JOB_MASTER][T0][YB42C0A89F89-00062963DE235926-0-0] [lt=25] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1734400173530721, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-16 17:49:28.530801] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [128532][DBMS_JOB_MASTER][T0][YB42C0A89F89-00062963DE235926-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1734400173530797, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-16 17:49:28.538148] INFO [RS] do_work (ob_common_ls_service.cpp:138) [128656][T1_COMMONLSSe][T1][YB42C0A89F89-00062963E043FA77-0-0] [lt=35] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:28.656521] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128853][T1_L0_G0][T1][YB42C0A89F8A-00062963E0A9757F-0-0] [lt=23] [DDL] execute ddl like stmt(ret=0, cost=13, ddl_arg=NULL) [2024-12-16 17:49:28.698911] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [129880][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=27] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1734400167727705, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1734400165433724004, replayable_scn:1734400165433724004, readable_scn:1734359204309426002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-16 17:49:28.735033] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.159.137:2882") [2024-12-16 17:49:28.735120] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=86] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="192.168.159.138:2882") [2024-12-16 17:49:28.735130] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="192.168.159.139:2882") [2024-12-16 17:49:28.735137] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=3, dest="192.168.159.141:2882") [2024-12-16 17:49:28.735144] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=4, dest="192.168.159.142:2882") [2024-12-16 17:49:28.735150] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=5, dest="192.168.159.143:2882") [2024-12-16 17:49:28.773295] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129974][T1001_StandbySc][T1001][YB42C0A89F89-00062963E5D3F94D-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:29.024277] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129809][T1002_StandbySc][T1002][YB42C0A89F89-00062963E533F94F-0-0] [lt=39] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:29.028694] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [128685][T1_HBService][T1][YB42C0A89F89-00062963E023AA0C-0-0] [lt=42] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:29.040792] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F89-00062963E103A9E2-0-0] [lt=45] [DDL] execute ddl like stmt(ret=0, cost=18, ddl_arg=NULL) [2024-12-16 17:49:29.049831] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F41F-0-0] [lt=23] is primary tenant, no need check readable_scn [2024-12-16 17:49:29.067944] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F41F-0-0] [lt=42] is primary tenant, no need check readable_scn [2024-12-16 17:49:29.076793] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F41F-0-0] [lt=25] is primary tenant, no need check readable_scn [2024-12-16 17:49:29.115453] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8F-000629642BDC7AD1-0-0] [lt=21] [DDL] execute ddl like stmt(ret=0, cost=25, ddl_arg=NULL) [2024-12-16 17:49:29.153371] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [128655][T1_StandbySchem][T1][YB42C0A89F89-00062963DD93FAC7-0-0] [lt=48] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:29.181694] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F7FA-0-0] [lt=45] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202081794000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202110029000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:0, operation_type_:0, create_scn:{val:1734359202119679000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:29.184711] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F7FA-0-0] [lt=129] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-16 17:49:29.184804] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F7FA-0-0] [lt=92] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:29.308954] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8A-00062963DFD99B26-0-0] [lt=34] [DDL] execute ddl like stmt(ret=0, cost=19, ddl_arg=NULL) [2024-12-16 17:49:29.366275] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8D-000629641B98C907-0-0] [lt=30] [DDL] execute ddl like stmt(ret=0, cost=21, ddl_arg=NULL) [2024-12-16 17:49:29.434432] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F410-0-0] [lt=36] [COMMON_LS_SERVICE] start to create ls(info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, create_scn={val:1734359202119679000, v:0}) [2024-12-16 17:49:29.448068] INFO [RS.LB] idle (ob_root_balancer.cpp:103) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D759-0-0] [lt=23] root balance idle(idle_time=10000000) [2024-12-16 17:49:29.448100] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2074) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=22] start try disaster recovery [2024-12-16 17:49:29.448112] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=9] start try tenant disaster recovery(tenant_id=1, only_for_display=false) [2024-12-16 17:49:29.450156] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=8] start try ls disaster recovery(ls_status_info={tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-16 17:49:29.465026] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=25] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1, ls_id={id:1}, acc_dr_task=0) [2024-12-16 17:49:29.465081] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=43] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:29.465094] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:1734358823275341, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.159.137:2882", sql_port:2881, role:1, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1734358823287139, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.159.138:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1734358823290920, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.159.139:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2024-12-16 17:49:29.465590] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=487] build ls locality stat map(tenant_id=1, ls_id={id:1}) [2024-12-16 17:49:29.465613] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=16] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-16 17:49:29.465627] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=13] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-16 17:49:29.465758] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=127] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-16 17:49:29.465777] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=13] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1734358823275341, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.159.137:2882", sql_port:2881, role:1, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1734358823287139, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.159.138:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1734358823290920, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.159.139:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-16 17:49:29.466015] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=225] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:29.466160] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=11] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:29.466177] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=14] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:29.466185] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:29.466345] WDIAG [RS] do_create_user_ls (ob_common_ls_service.cpp:269) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F410-0-0] [lt=10][errcode=-4672] failed to create user ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, locality_array=[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], create_scn={val:1734359202119679000, v:0}, palf_base_info={prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}, curr_lsn:{lsn:18446744073709551615}}, create_with_palf=false, source_tenant_id=0) [2024-12-16 17:49:29.466418] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:274) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F410-0-0] [lt=60] [COMMON_LS_SERVICE] end to create ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, cost=31939) [2024-12-16 17:49:29.466453] WDIAG [RS] try_create_ls_ (ob_common_ls_service.cpp:179) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F410-0-0] [lt=34][errcode=-4672] failed to create new ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", status_info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, recovery_stat={tenant_id:1002, ls_id:{id:1003}, sync_scn:{val:1, v:0}, readable_scn:{val:1, v:0}, create_scn:{val:1734359202119679000, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:9223372036854775807, config_seq:-1}}) [2024-12-16 17:49:29.466491] WDIAG [RS] do_work (ob_common_ls_service.cpp:117) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F410-0-0] [lt=33][errcode=0] failed to create ls(ret=0, ret="OB_SUCCESS", tmp_ret=-4672, tmp_ret="OB_REPLICA_NUM_NOT_ENOUGH", user_tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:29.467825] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=6] try tenant disaster recovery succ(tenant_id=1, only_for_display=false) [2024-12-16 17:49:29.468107] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=279] start try tenant disaster recovery(tenant_id=1001, only_for_display=false) [2024-12-16 17:49:29.469527] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=15] start try ls disaster recovery(ls_status_info={tenant_id:1001, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-16 17:49:29.472432] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F410-0-0] [lt=61] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}, {unit_group_id:1002, unit_status:0, ls_group_ids:[]}]) [2024-12-16 17:49:29.482959] INFO [RS] do_work (ob_common_ls_service.cpp:138) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F410-0-0] [lt=27] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:29.484402] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=73] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1001, ls_id={id:1}, acc_dr_task=0) [2024-12-16 17:49:29.484435] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=26] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:29.484448] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1001, ls_id:{id:1}, replicas:[{modify_time_us:1734359140223858, create_time_us:1734359134691166, tenant_id:1001, ls_id:{id:1}, server:"192.168.159.137:2882", sql_port:2881, role:1, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1734359134832984, create_time_us:1734359134670339, tenant_id:1001, ls_id:{id:1}, server:"192.168.159.138:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1734359135157794, create_time_us:1734359134696290, tenant_id:1001, ls_id:{id:1}, server:"192.168.159.139:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1001, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2024-12-16 17:49:29.484594] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=142] build ls locality stat map(tenant_id=1001, ls_id={id:1}) [2024-12-16 17:49:29.484607] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-16 17:49:29.484621] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=12] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-16 17:49:29.484630] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-16 17:49:29.484641] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1734359140223858, create_time_us:1734359134691166, tenant_id:1001, ls_id:{id:1}, server:"192.168.159.137:2882", sql_port:2881, role:1, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1734359134832984, create_time_us:1734359134670339, tenant_id:1001, ls_id:{id:1}, server:"192.168.159.138:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1734359135157794, create_time_us:1734359134696290, tenant_id:1001, ls_id:{id:1}, server:"192.168.159.139:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-16 17:49:29.484927] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=271] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:29.484947] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=10] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:29.484957] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:29.484965] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:29.487786] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=6] try tenant disaster recovery succ(tenant_id=1001, only_for_display=false) [2024-12-16 17:49:29.489067] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=19] start try tenant disaster recovery(tenant_id=1002, only_for_display=false) [2024-12-16 17:49:29.490300] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=16] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-16 17:49:29.502551] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=24] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1}, acc_dr_task=0) [2024-12-16 17:49:29.502586] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=27] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:29.502599] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=11] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1}, replicas:[{modify_time_us:1734359175153676, create_time_us:1734359170239483, tenant_id:1002, ls_id:{id:1}, server:"192.168.159.137:2882", sql_port:2881, role:1, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1734359171015694, create_time_us:1734359169618664, tenant_id:1002, ls_id:{id:1}, server:"192.168.159.138:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1734359170930591, create_time_us:1734359169573173, tenant_id:1002, ls_id:{id:1}, server:"192.168.159.139:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2024-12-16 17:49:29.502855] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=249] build ls locality stat map(tenant_id=1002, ls_id={id:1}) [2024-12-16 17:49:29.502870] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-16 17:49:29.502883] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=13] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-16 17:49:29.502893] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-16 17:49:29.502904] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1734359175153676, create_time_us:1734359170239483, tenant_id:1002, ls_id:{id:1}, server:"192.168.159.137:2882", sql_port:2881, role:1, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1734359171015694, create_time_us:1734359169618664, tenant_id:1002, ls_id:{id:1}, server:"192.168.159.138:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1734359170930591, create_time_us:1734359169573173, tenant_id:1002, ls_id:{id:1}, server:"192.168.159.139:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-16 17:49:29.503403] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=484] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:29.503426] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=12] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:29.503437] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:29.503445] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:29.505556] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-16 17:49:29.533792] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=25] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1001}, acc_dr_task=0) [2024-12-16 17:49:29.533826] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=24] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:29.533837] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1001}, replicas:[{modify_time_us:1734359215801871, create_time_us:1734359210352363, tenant_id:1002, ls_id:{id:1001}, server:"192.168.159.137:2882", sql_port:2881, role:1, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1734359211537200, create_time_us:1734359210337910, tenant_id:1002, ls_id:{id:1001}, server:"192.168.159.138:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1734359211479956, create_time_us:1734359210323237, tenant_id:1002, ls_id:{id:1001}, server:"192.168.159.139:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2024-12-16 17:49:29.534063] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=219] build ls locality stat map(tenant_id=1002, ls_id={id:1001}) [2024-12-16 17:49:29.534082] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=14] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-16 17:49:29.534096] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=13] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-16 17:49:29.534105] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-16 17:49:29.534117] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1734359215801871, create_time_us:1734359210352363, tenant_id:1002, ls_id:{id:1001}, server:"192.168.159.137:2882", sql_port:2881, role:1, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1734359211537200, create_time_us:1734359210337910, tenant_id:1002, ls_id:{id:1001}, server:"192.168.159.138:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1734359211479956, create_time_us:1734359210323237, tenant_id:1002, ls_id:{id:1001}, server:"192.168.159.139:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-16 17:49:29.534231] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=103] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:29.534249] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:29.534259] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:29.534267] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:29.537098] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-16 17:49:29.547400] INFO [RS] do_work (ob_common_ls_service.cpp:138) [128656][T1_COMMONLSSe][T1][YB42C0A89F89-00062963E043FA78-0-0] [lt=26] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:29.556358] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=51] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1002}, acc_dr_task=0) [2024-12-16 17:49:29.556396] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=28] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:29.556409] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1002}, replicas:[{modify_time_us:1734359211546178, create_time_us:1734359210734985, tenant_id:1002, ls_id:{id:1002}, server:"192.168.159.137:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1734359216339659, create_time_us:1734359210724652, tenant_id:1002, ls_id:{id:1002}, server:"192.168.159.138:2882", sql_port:2881, role:1, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1734359211505782, create_time_us:1734359210727004, tenant_id:1002, ls_id:{id:1002}, server:"192.168.159.139:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2024-12-16 17:49:29.556555] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=141] build ls locality stat map(tenant_id=1002, ls_id={id:1002}) [2024-12-16 17:49:29.556568] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-16 17:49:29.556581] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=13] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-16 17:49:29.556591] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-16 17:49:29.556602] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1734359211546178, create_time_us:1734359210734985, tenant_id:1002, ls_id:{id:1002}, server:"192.168.159.137:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1734359216339659, create_time_us:1734359210724652, tenant_id:1002, ls_id:{id:1002}, server:"192.168.159.138:2882", sql_port:2881, role:1, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1734359211505782, create_time_us:1734359210727004, tenant_id:1002, ls_id:{id:1002}, server:"192.168.159.139:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-16 17:49:29.556791] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=176] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:29.556810] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:29.556820] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:29.556828] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:29.558772] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-16 17:49:29.558800] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=24] try tenant disaster recovery succ(tenant_id=1002, only_for_display=false) [2024-12-16 17:49:29.558812] INFO [RS] check (ob_migrate_unit_finish_checker.cpp:79) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=8] start check unit migrate finish [2024-12-16 17:49:29.558826] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=8] try check migrate unit finish by tenant(tenant_id=1) [2024-12-16 17:49:29.567372] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=7] try check migrate unit finish by tenant(tenant_id=1002) [2024-12-16 17:49:29.586784] WDIAG [SHARE.PT] find_leader (ob_ls_info.cpp:848) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=48][errcode=-4018] fail to get leader replica(ret=-4018, ret="OB_ENTRY_NOT_EXIST", *this={tenant_id:1002, ls_id:{id:1003}, replicas:[]}, replica count=0) [2024-12-16 17:49:29.586865] WDIAG [RS] construct_filtered_ls_info_to_use_ (ob_disaster_recovery_info.cpp:398) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=78][errcode=-4018] fail to find leader replica in input_ls_info(ret=-4018, ret="OB_ENTRY_NOT_EXIST", input_ls_info={tenant_id:1002, ls_id:{id:1003}, replicas:[]}) [2024-12-16 17:49:29.586887] WDIAG [RS] build_disaster_ls_info (ob_disaster_recovery_info.cpp:459) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=20][errcode=-4018] fail to filter not in member/learner list replicas and learner_with_flag replicas(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_info={tenant_id:1002, ls_id:{id:1003}, replicas:[]}, filter_readonly_replicas_with_flag=true) [2024-12-16 17:49:29.586899] WDIAG [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:200) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=11][errcode=-4018] fail to generate dr log stream info(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-16 17:49:29.587757] WDIAG [RS] check (ob_migrate_unit_finish_checker.cpp:96) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=6][errcode=0] fail to try check migrate unit finish by tenant(tmp_ret=-4018, tmp_ret="OB_ENTRY_NOT_EXIST", tenant_id=1002) [2024-12-16 17:49:29.587865] INFO [RS] check (ob_alter_locality_finish_checker.cpp:79) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75A-0-0] [lt=67] start to check alter locality finish [2024-12-16 17:49:29.589024] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75B-0-0] [lt=45][errcode=-4008] empty result set(ret=-4008) [2024-12-16 17:49:29.590179] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75B-0-0] [lt=38][errcode=-4008] empty result set(ret=-4008) [2024-12-16 17:49:29.592546] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75D-0-0] [lt=37][errcode=-4008] empty result set(ret=-4008) [2024-12-16 17:49:29.593997] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75D-0-0] [lt=15][errcode=-4008] empty result set(ret=-4008) [2024-12-16 17:49:29.595345] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:72) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75D-0-0] [lt=16] start check shrink resource pool [2024-12-16 17:49:29.595412] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75E-0-0] [lt=59] start check shrink resource pool(tenant_id=1) [2024-12-16 17:49:29.597014] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75E-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2024-12-16 17:49:29.598431] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75E-0-0] [lt=35][errcode=-4008] empty result set(ret=-4008) [2024-12-16 17:49:29.598652] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75F-0-0] [lt=140] start check shrink resource pool(tenant_id=1002) [2024-12-16 17:49:29.609736] INFO [SHARE] get_balance_job (ob_balance_job_table_operator.cpp:209) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75F-0-0] [lt=738] empty balance job(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sql=select time_to_usec(gmt_create) as start_time, time_to_usec(gmt_modified) as finish_time, * from __all_balance_job) [2024-12-16 17:49:29.612291] WDIAG [BALANCE] gather_ls_status_stat (ob_tenant_balance_service.cpp:264) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75F-0-0] [lt=90][errcode=-4076] [TENANT_BALANCE] ls status not ready, can not balance(ret=-4076, ret="OB_NEED_WAIT", ls_info={id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:0, operation_type_:0, create_scn:{val:1734359202119679000, v:0}}, status_info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-16 17:49:29.612355] WDIAG [BALANCE] is_primary_tenant_ls_balance_finished_ (ob_tenant_balance_service.cpp:366) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75F-0-0] [lt=60][errcode=-4076] fail to execute gather_ls_status_stat(ret=-4076, ret="OB_NEED_WAIT", tenant_id=1002) [2024-12-16 17:49:29.612369] INFO [BALANCE] is_primary_tenant_ls_balance_finished_ (ob_tenant_balance_service.cpp:379) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75F-0-0] [lt=13] check whether the primary_tenant has balanced ls(ret=-4076, ret="OB_NEED_WAIT", tenant_id=1002, ls_array=[{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, {tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, {tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, {tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}], primary_zone_num=0, unit_group_array=[], need_ls_balance=false) [2024-12-16 17:49:29.612408] WDIAG begin (ob_hashtable.h:932) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75F-0-0] [lt=28][errcode=-4006] hashtable not init, backtrace=0x7bcec65 0xe3d6410 0xe3c663b 0xce43adb 0xce40eb5 0xc693a3d 0xc822979 0xcf8b6dc 0xc7e823a 0xc7e4fbd 0xc7e49a4 0x1f623704 0x1f629f7c 0x1f629758 0x1f6278f7 0x2b456c0fbea5 0x2b456c40e96d [2024-12-16 17:49:29.612417] WDIAG [BALANCE] is_ls_balance_finished (ob_tenant_balance_service.cpp:315) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75F-0-0] [lt=7][errcode=-4076] fail to execute is_primary_tenant_ls_balance_finished_(ret=-4076, ret="OB_NEED_WAIT", tenant_id=1002) [2024-12-16 17:49:29.612425] WDIAG [RS] check_tenant_ls_balance (ob_root_utils.cpp:2059) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75F-0-0] [lt=6][errcode=-4076] fail to execute is_ls_balance_finished(ret=-4076, ret="OB_NEED_WAIT", tenant_id=1002) [2024-12-16 17:49:29.612432] WDIAG [RS] check_ls_balance_and_commit_rs_job (ob_root_utils.cpp:2123) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75F-0-0] [lt=6][errcode=-4076] fail to execute check_tenant_ls_balance(ret=-4076, ret="OB_NEED_WAIT", tenant_id=1002) [2024-12-16 17:49:29.612437] WDIAG [RS] check_shrink_resource_pool_finished_by_tenant_ (ob_shrink_expand_resource_pool_checker.cpp:148) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75F-0-0] [lt=5][errcode=-4076] fail to execute check_ls_balance_and_commit_rs_job(ret=-4076, ret="OB_NEED_WAIT", tenant_id=1002) [2024-12-16 17:49:29.612445] WDIAG [RS] check (ob_shrink_expand_resource_pool_checker.cpp:95) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75F-0-0] [lt=6][errcode=0] fail to check shrink resource pool finish(ret=0, ret="OB_SUCCESS", tmp_ret=-4076, tmp_ret="OB_NEED_WAIT", tenant_id=1002) [2024-12-16 17:49:29.612455] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:100) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75F-0-0] [lt=8] finish check shrink resource pool(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:29.612463] INFO [RS] check (ob_alter_primary_zone_checker.cpp:49) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D75F-0-0] [lt=5] start check primary zone [2024-12-16 17:49:29.612477] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D760-0-0] [lt=4] start check primary zone(tenant_id=1) [2024-12-16 17:49:29.613539] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D760-0-0] [lt=6][errcode=-4008] empty result set(ret=-4008) [2024-12-16 17:49:29.613718] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=49] start check primary zone(tenant_id=1002) [2024-12-16 17:49:29.614766] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=34][errcode=-4008] empty result set(ret=-4008) [2024-12-16 17:49:29.614864] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:179) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=36] start do unit balance [2024-12-16 17:49:29.616060] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [128528][RootBalance][T1][YB42C0A89F89-00062963E283D761-0-0] [lt=14] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:12274, add_timestamp:1734400169616027}) [2024-12-16 17:49:29.626095] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128855][T1_L0_G0][T1][YB42C0A89F89-00062963E2039F00-0-0] [lt=36] [DDL] execute ddl like stmt(ret=0, cost=11, ddl_arg=NULL) [2024-12-16 17:49:29.647098] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:363) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=271] start distribute for server status change [2024-12-16 17:49:29.647186] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:414) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=24] finish distribute for server status change(ret=0) [2024-12-16 17:49:29.647199] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:213) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=13] finish do unit balance(ret=0) [2024-12-16 17:49:29.665102] INFO [RS.LB] tenant_group_balance (ob_server_balancer.cpp:128) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=76] start to do tenant group unit balance [2024-12-16 17:49:29.665165] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:930) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=27] start to do tenant group unit balance(zone=zone1) [2024-12-16 17:49:29.665186] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7935) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=12] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone1", server_disk_statistic_array_:[{server_:"192.168.159.141:2882", disk_in_use_:0, disk_total_:8589934592, wild_server_:false}, {server_:"192.168.159.137:2882", disk_in_use_:287309824, disk_total_:8589934592, wild_server_:false}], over_disk_waterlevel_:false}) [2024-12-16 17:49:29.665212] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1084) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=19] generate tenant array finish(ret=0, zone=zone1, standalone_tenant_array=[1001, 1002], group_tenant_array=[]) [2024-12-16 17:49:29.665332] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=86] generate available servers finish(ret=0, zone=zone1, available_servers=["192.168.159.137:2882", "192.168.159.141:2882"]) [2024-12-16 17:49:29.665372] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=22] generate available servers finish(ret=0, zone=zone1, available_servers=["192.168.159.137:2882", "192.168.159.141:2882"]) [2024-12-16 17:49:29.665408] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5311) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=31] start do non-tenantgroup unit balance task(zone=zone1) [2024-12-16 17:49:29.665428] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5640) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=11] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone1, cpu_weights=2.429022082076017497e-01, mem_weights=6.056782334527212042e-01, log_disk_weights=1.514195583396769074e-01, data_disk_weights=0.000000000000000000e+00, available_servers=["192.168.159.137:2882", "192.168.159.141:2882"]) [2024-12-16 17:49:29.665462] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4547) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=21] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"192.168.159.137:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:7, max_cpu:7, memory_size:"7GB", log_disk_size:"4GB", data_disk_size:0, min_iops:-2, max_iops:-2, iops_weight:6, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:7, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:7, cpu_assigned_max:7, mem_capacity:"7GB", mem_assigned:"7GB", mem_in_use:0GB, log_disk_capacity:14GB, log_disk_assigned:4GB, log_disk_in_use:1.375GB, data_disk_capacity:8GB, data_disk_assigned:0GB, data_disk_in_use:0.267578125GB}, inter_ttg_load_value_:"7.552106804876674850e-01"}, {server_:"192.168.159.141:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:4, max_cpu:4, memory_size:"5GB", log_disk_size:"2GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:3, max_net_bandwidth:"-1.86264515e-09GB", net_bandwidth_weight:4, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:4, cpu_assigned_max:4, mem_capacity:"7GB", mem_assigned:"5GB", mem_in_use:0GB, log_disk_capacity:14GB, log_disk_assigned:2GB, log_disk_in_use:0GB, data_disk_capacity:8GB, data_disk_assigned:0GB, data_disk_in_use:0GB}, inter_ttg_load_value_:"5.149842271280108408e-01"}], available_servers=["192.168.159.137:2882", "192.168.159.141:2882"], wild_server_loads=[], over_server_loads=[{server_:"192.168.159.137:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:7, max_cpu:7, memory_size:"7GB", log_disk_size:"4GB", data_disk_size:0, min_iops:-2, max_iops:-2, iops_weight:6, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:7, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:7, cpu_assigned_max:7, mem_capacity:"7GB", mem_assigned:"7GB", mem_in_use:0GB, log_disk_capacity:14GB, log_disk_assigned:4GB, log_disk_in_use:1.375GB, data_disk_capacity:8GB, data_disk_assigned:0GB, data_disk_in_use:0.267578125GB}, inter_ttg_load_value_:"7.552106804876674850e-01"}], under_server_loads=[{server_:"192.168.159.141:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:4, max_cpu:4, memory_size:"5GB", log_disk_size:"2GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:3, max_net_bandwidth:"-1.86264515e-09GB", net_bandwidth_weight:4, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:4, cpu_assigned_max:4, mem_capacity:"7GB", mem_assigned:"5GB", mem_in_use:0GB, log_disk_capacity:14GB, log_disk_assigned:2GB, log_disk_in_use:0GB, data_disk_capacity:8GB, data_disk_assigned:0GB, data_disk_in_use:0GB}, inter_ttg_load_value_:"5.149842271280108408e-01"}]) [2024-12-16 17:49:29.665537] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8126) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=74] check server over disk waterlevel(zone=zone1, disk_statistic={server_:"192.168.159.141:2882", disk_in_use_:0, disk_total_:8589934592, wild_server_:false}, over_disk_waterlevel=false) [2024-12-16 17:49:29.665603] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8132) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=64] check all available servers over disk waterlevel(zone=zone1, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"192.168.159.141:2882", disk_in_use_:0, disk_total_:8589934592, wild_server_:false}, {server_:"192.168.159.137:2882", disk_in_use_:287309824, disk_total_:8589934592, wild_server_:false}]) [2024-12-16 17:49:29.665715] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5293) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=100] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=2, not_grant_units.count()=0) [2024-12-16 17:49:29.665733] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5348) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=16] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone1, upper_lmt=6.668523264982311183e-01, wild_server_loads.count()=0, over_server_loads.count()=1, under_server_loads.count()=1, standalone_units=[{unit_id:1001, tenant_id:1002, resource_pool_id:1001, unit_config:{unit_config_id:1001, name:"unit_1", resource:{min_cpu:4, max_cpu:4, memory_size:"5GB", log_disk_size:"2GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x2b45bca19b38, unit_config:0x2b45be5f7d18, pool:0x2b4615fe5688}, {unit_id:1004, tenant_id:1002, resource_pool_id:1001, unit_config:{unit_config_id:1001, name:"unit_1", resource:{min_cpu:4, max_cpu:4, memory_size:"5GB", log_disk_size:"2GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x2b45bca198b0, unit_config:0x2b45be5f7d18, pool:0x2b4615fe5688}], not_grant_units=[], available_servers=["192.168.159.137:2882", "192.168.159.141:2882"]) [2024-12-16 17:49:29.665831] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:930) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=28] start to do tenant group unit balance(zone=zone2) [2024-12-16 17:49:29.665852] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7935) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=13] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone2", server_disk_statistic_array_:[{server_:"192.168.159.142:2882", disk_in_use_:0, disk_total_:8589934592, wild_server_:false}, {server_:"192.168.159.138:2882", disk_in_use_:251658240, disk_total_:8589934592, wild_server_:false}], over_disk_waterlevel_:false}) [2024-12-16 17:49:29.665872] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1084) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=13] generate tenant array finish(ret=0, zone=zone2, standalone_tenant_array=[1001, 1002], group_tenant_array=[]) [2024-12-16 17:49:29.665964] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=74] generate available servers finish(ret=0, zone=zone2, available_servers=["192.168.159.138:2882", "192.168.159.142:2882"]) [2024-12-16 17:49:29.665990] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=15] generate available servers finish(ret=0, zone=zone2, available_servers=["192.168.159.138:2882", "192.168.159.142:2882"]) [2024-12-16 17:49:29.666001] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5311) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=9] start do non-tenantgroup unit balance task(zone=zone2) [2024-12-16 17:49:29.666012] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5640) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=6] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone2, cpu_weights=2.429022082076017497e-01, mem_weights=6.056782334527212042e-01, log_disk_weights=1.514195583396769074e-01, data_disk_weights=0.000000000000000000e+00, available_servers=["192.168.159.138:2882", "192.168.159.142:2882"]) [2024-12-16 17:49:29.666032] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4547) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=14] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"192.168.159.138:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:7, max_cpu:7, memory_size:"7GB", log_disk_size:"4GB", data_disk_size:0, min_iops:-2, max_iops:-2, iops_weight:6, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:7, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:7, cpu_assigned_max:7, mem_capacity:"7GB", mem_assigned:"7GB", mem_in_use:0GB, log_disk_capacity:14GB, log_disk_assigned:4GB, log_disk_in_use:1.375GB, data_disk_capacity:8GB, data_disk_assigned:0GB, data_disk_in_use:0.234375GB}, inter_ttg_load_value_:"7.552106804876674850e-01"}, {server_:"192.168.159.142:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:4, max_cpu:4, memory_size:"5GB", log_disk_size:"2GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:3, max_net_bandwidth:"-1.86264515e-09GB", net_bandwidth_weight:4, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:4, cpu_assigned_max:4, mem_capacity:"7GB", mem_assigned:"5GB", mem_in_use:0GB, log_disk_capacity:14GB, log_disk_assigned:2GB, log_disk_in_use:0GB, data_disk_capacity:8GB, data_disk_assigned:0GB, data_disk_in_use:0GB}, inter_ttg_load_value_:"5.149842271280108408e-01"}], available_servers=["192.168.159.138:2882", "192.168.159.142:2882"], wild_server_loads=[], over_server_loads=[{server_:"192.168.159.138:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:7, max_cpu:7, memory_size:"7GB", log_disk_size:"4GB", data_disk_size:0, min_iops:-2, max_iops:-2, iops_weight:6, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:7, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:7, cpu_assigned_max:7, mem_capacity:"7GB", mem_assigned:"7GB", mem_in_use:0GB, log_disk_capacity:14GB, log_disk_assigned:4GB, log_disk_in_use:1.375GB, data_disk_capacity:8GB, data_disk_assigned:0GB, data_disk_in_use:0.234375GB}, inter_ttg_load_value_:"7.552106804876674850e-01"}], under_server_loads=[{server_:"192.168.159.142:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:4, max_cpu:4, memory_size:"5GB", log_disk_size:"2GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:3, max_net_bandwidth:"-1.86264515e-09GB", net_bandwidth_weight:4, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:4, cpu_assigned_max:4, mem_capacity:"7GB", mem_assigned:"5GB", mem_in_use:0GB, log_disk_capacity:14GB, log_disk_assigned:2GB, log_disk_in_use:0GB, data_disk_capacity:8GB, data_disk_assigned:0GB, data_disk_in_use:0GB}, inter_ttg_load_value_:"5.149842271280108408e-01"}]) [2024-12-16 17:49:29.666086] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8126) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=52] check server over disk waterlevel(zone=zone2, disk_statistic={server_:"192.168.159.142:2882", disk_in_use_:0, disk_total_:8589934592, wild_server_:false}, over_disk_waterlevel=false) [2024-12-16 17:49:29.666095] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8132) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=9] check all available servers over disk waterlevel(zone=zone2, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"192.168.159.142:2882", disk_in_use_:0, disk_total_:8589934592, wild_server_:false}, {server_:"192.168.159.138:2882", disk_in_use_:251658240, disk_total_:8589934592, wild_server_:false}]) [2024-12-16 17:49:29.666111] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5293) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=11] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=2, not_grant_units.count()=0) [2024-12-16 17:49:29.666123] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5348) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=11] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone2, upper_lmt=6.668523264982311183e-01, wild_server_loads.count()=0, over_server_loads.count()=1, under_server_loads.count()=1, standalone_units=[{unit_id:1002, tenant_id:1002, resource_pool_id:1001, unit_config:{unit_config_id:1001, name:"unit_1", resource:{min_cpu:4, max_cpu:4, memory_size:"5GB", log_disk_size:"2GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x2b45bca19a60, unit_config:0x2b45be5f7d18, pool:0x2b4615fe5688}, {unit_id:1005, tenant_id:1002, resource_pool_id:1001, unit_config:{unit_config_id:1001, name:"unit_1", resource:{min_cpu:4, max_cpu:4, memory_size:"5GB", log_disk_size:"2GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x2b45bca197d8, unit_config:0x2b45be5f7d18, pool:0x2b4615fe5688}], not_grant_units=[], available_servers=["192.168.159.138:2882", "192.168.159.142:2882"]) [2024-12-16 17:49:29.666164] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:930) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=24] start to do tenant group unit balance(zone=zone3) [2024-12-16 17:49:29.666177] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7935) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=8] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone3", server_disk_statistic_array_:[{server_:"192.168.159.143:2882", disk_in_use_:0, disk_total_:8589934592, wild_server_:false}, {server_:"192.168.159.139:2882", disk_in_use_:266338304, disk_total_:8589934592, wild_server_:false}], over_disk_waterlevel_:false}) [2024-12-16 17:49:29.666194] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1084) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=12] generate tenant array finish(ret=0, zone=zone3, standalone_tenant_array=[1001, 1002], group_tenant_array=[]) [2024-12-16 17:49:29.666214] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=8] generate available servers finish(ret=0, zone=zone3, available_servers=["192.168.159.139:2882", "192.168.159.143:2882"]) [2024-12-16 17:49:29.666233] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=9] generate available servers finish(ret=0, zone=zone3, available_servers=["192.168.159.139:2882", "192.168.159.143:2882"]) [2024-12-16 17:49:29.666244] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5311) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=9] start do non-tenantgroup unit balance task(zone=zone3) [2024-12-16 17:49:29.666253] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5640) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=6] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone3, cpu_weights=2.429022082076017497e-01, mem_weights=6.056782334527212042e-01, log_disk_weights=1.514195583396769074e-01, data_disk_weights=0.000000000000000000e+00, available_servers=["192.168.159.139:2882", "192.168.159.143:2882"]) [2024-12-16 17:49:29.666272] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4547) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=13] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"192.168.159.139:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:7, max_cpu:7, memory_size:"7GB", log_disk_size:"4GB", data_disk_size:0, min_iops:-2, max_iops:-2, iops_weight:6, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:7, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:7, cpu_assigned_max:7, mem_capacity:"7GB", mem_assigned:"7GB", mem_in_use:0GB, log_disk_capacity:14GB, log_disk_assigned:4GB, log_disk_in_use:1.375GB, data_disk_capacity:8GB, data_disk_assigned:0GB, data_disk_in_use:0.248046875GB}, inter_ttg_load_value_:"7.552106804876674850e-01"}, {server_:"192.168.159.143:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:4, max_cpu:4, memory_size:"5GB", log_disk_size:"2GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:3, max_net_bandwidth:"-1.86264515e-09GB", net_bandwidth_weight:4, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:4, cpu_assigned_max:4, mem_capacity:"7GB", mem_assigned:"5GB", mem_in_use:0GB, log_disk_capacity:14GB, log_disk_assigned:2GB, log_disk_in_use:0GB, data_disk_capacity:8GB, data_disk_assigned:0GB, data_disk_in_use:0GB}, inter_ttg_load_value_:"5.149842271280108408e-01"}], available_servers=["192.168.159.139:2882", "192.168.159.143:2882"], wild_server_loads=[], over_server_loads=[{server_:"192.168.159.139:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:7, max_cpu:7, memory_size:"7GB", log_disk_size:"4GB", data_disk_size:0, min_iops:-2, max_iops:-2, iops_weight:6, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:7, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:7, cpu_assigned_max:7, mem_capacity:"7GB", mem_assigned:"7GB", mem_in_use:0GB, log_disk_capacity:14GB, log_disk_assigned:4GB, log_disk_in_use:1.375GB, data_disk_capacity:8GB, data_disk_assigned:0GB, data_disk_in_use:0.248046875GB}, inter_ttg_load_value_:"7.552106804876674850e-01"}], under_server_loads=[{server_:"192.168.159.143:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:4, max_cpu:4, memory_size:"5GB", log_disk_size:"2GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:3, max_net_bandwidth:"-1.86264515e-09GB", net_bandwidth_weight:4, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:4, cpu_assigned_max:4, mem_capacity:"7GB", mem_assigned:"5GB", mem_in_use:0GB, log_disk_capacity:14GB, log_disk_assigned:2GB, log_disk_in_use:0GB, data_disk_capacity:8GB, data_disk_assigned:0GB, data_disk_in_use:0GB}, inter_ttg_load_value_:"5.149842271280108408e-01"}]) [2024-12-16 17:49:29.666323] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8126) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=51] check server over disk waterlevel(zone=zone3, disk_statistic={server_:"192.168.159.143:2882", disk_in_use_:0, disk_total_:8589934592, wild_server_:false}, over_disk_waterlevel=false) [2024-12-16 17:49:29.666332] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8132) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=9] check all available servers over disk waterlevel(zone=zone3, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"192.168.159.143:2882", disk_in_use_:0, disk_total_:8589934592, wild_server_:false}, {server_:"192.168.159.139:2882", disk_in_use_:266338304, disk_total_:8589934592, wild_server_:false}]) [2024-12-16 17:49:29.666347] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5293) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=10] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=2, not_grant_units.count()=0) [2024-12-16 17:49:29.666358] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5348) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=10] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone3, upper_lmt=6.668523264982311183e-01, wild_server_loads.count()=0, over_server_loads.count()=1, under_server_loads.count()=1, standalone_units=[{unit_id:1003, tenant_id:1002, resource_pool_id:1001, unit_config:{unit_config_id:1001, name:"unit_1", resource:{min_cpu:4, max_cpu:4, memory_size:"5GB", log_disk_size:"2GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x2b45bca19988, unit_config:0x2b45be5f7d18, pool:0x2b4615fe5688}, {unit_id:1006, tenant_id:1002, resource_pool_id:1001, unit_config:{unit_config_id:1001, name:"unit_1", resource:{min_cpu:4, max_cpu:4, memory_size:"5GB", log_disk_size:"2GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x2b45bca19700, unit_config:0x2b45be5f7d18, pool:0x2b4615fe5688}], not_grant_units=[], available_servers=["192.168.159.139:2882", "192.168.159.143:2882"]) [2024-12-16 17:49:29.705610] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [128684][T1_HBService][T1][YB42C0A89F89-00062963E003A9C0-0-0] [lt=183] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:29.736359] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=27] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.159.137:2882") [2024-12-16 17:49:29.736399] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=38] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="192.168.159.138:2882") [2024-12-16 17:49:29.736411] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="192.168.159.139:2882") [2024-12-16 17:49:29.736418] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=3, dest="192.168.159.141:2882") [2024-12-16 17:49:29.736425] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=4, dest="192.168.159.142:2882") [2024-12-16 17:49:29.736432] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=5, dest="192.168.159.143:2882") [2024-12-16 17:49:29.736445] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=4226, return_code_array=[0, 0, 0, 0, 0, 0], broadcast_times=1) [2024-12-16 17:49:29.773504] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129974][T1001_StandbySc][T1001][YB42C0A89F89-00062963E5D3F94E-0-0] [lt=31] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:29.889261] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128852][T1_L0_G0][T1][YB42C0A89F8E-0006296421A04F3F-0-0] [lt=17] [DDL] execute ddl like stmt(ret=0, cost=10, ddl_arg=NULL) [2024-12-16 17:49:29.899977] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8B-00062963DFDFA01C-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=20, ddl_arg=NULL) [2024-12-16 17:49:29.904747] INFO [RS] run2 (ob_archive_scheduler_service.cpp:85) [128666][T1_ArchiveSvr][T1][YB42C0A89F89-00062963DFE35C63-0-0] [lt=9] start do ObArchiveSchedulerService round(round=830) [2024-12-16 17:49:29.907403] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:954) [128666][T1_ArchiveSvr][T1][YB42C0A89F89-00062963DFE35C63-0-0] [lt=34][errcode=-4018] no row exist(ret=-4018) [2024-12-16 17:49:29.907422] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:627) [128666][T1_ArchiveSvr][T1][YB42C0A89F89-00062963DFE35C63-0-0] [lt=17][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1 AND dest_no = 0 AND name = 'state', exec_tenant_id=1) [2024-12-16 17:49:29.907478] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [128666][T1_ArchiveSvr][T1][YB42C0A89F89-00062963DFE35C63-0-0] [lt=10][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-12-16 17:49:29.907500] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [128666][T1_ArchiveSvr][T1][YB42C0A89F89-00062963DFE35C63-0-0] [lt=20][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-12-16 17:49:29.907510] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [128666][T1_ArchiveSvr][T1][YB42C0A89F89-00062963DFE35C63-0-0] [lt=10][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-12-16 17:49:29.977851] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:717) [128664][T1_BackupDataSr][T1][YB42C0A89F89-00062963DFF35E8B-0-0] [lt=25] [DATA_BACKUP]success get jobs(job_attrs=[]) [2024-12-16 17:49:30.024378] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129809][T1002_StandbySc][T1002][YB42C0A89F89-00062963E533F950-0-0] [lt=31] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:30.025608] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8E-0006296420107351-0-0] [lt=31] [DDL] execute ddl like stmt(ret=0, cost=20, ddl_arg=NULL) [2024-12-16 17:49:30.089075] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F420-0-0] [lt=19] is primary tenant, no need check readable_scn [2024-12-16 17:49:30.110462] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F420-0-0] [lt=33] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.159.137:2882", readable_scn:{val:1734400169946989000, v:0}}, {server:"192.168.159.138:2882", readable_scn:{val:1734400169946989001, v:0}}, {server:"192.168.159.139:2882", readable_scn:{val:1734400169946989001, v:0}}], last_dump_ts_=1734400164870112, force_dump=false) [2024-12-16 17:49:30.110502] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F420-0-0] [lt=34] is primary tenant, no need check readable_scn [2024-12-16 17:49:30.129837] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F420-0-0] [lt=7] is primary tenant, no need check readable_scn [2024-12-16 17:49:30.156683] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [128655][T1_StandbySchem][T1][YB42C0A89F89-00062963DD93FAC8-0-0] [lt=26] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:30.189625] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F7FB-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202081794000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202110029000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:0, operation_type_:0, create_scn:{val:1734359202119679000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:30.192569] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F7FB-0-0] [lt=237] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-16 17:49:30.192592] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F7FB-0-0] [lt=22] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:30.246073] INFO [RS] check_lost_replicas (ob_lost_replica_checker.cpp:117) [128527][LostRepCheck][T0][Y0-0000000000000000-0-0] [lt=45] start checking lost replicas [2024-12-16 17:49:30.263144] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:131) [128661][T1_REST_SER][T1][YB42C0A89F89-00062963E053613A-0-0] [lt=17] [RESTORE] restore scheduler quit [2024-12-16 17:49:30.266389] INFO [SHARE] get_all_recover_table_job (ob_recover_table_persist_helper.cpp:199) [128661][T1_REST_SER][T1][YB42C0A89F89-00062963E053613A-0-0] [lt=53] get targets(ret=0, jobs=[], exec_tenant_id=1, sql=select * from __all_recover_table_job) [2024-12-16 17:49:30.266441] WDIAG [SHARE.SCHEMA] revert (ob_schema_mgr_cache.cpp:122) [128661][T1_REST_SER][T1][YB42C0A89F89-00062963E053613B-0-0] [lt=32][errcode=0] long time to hold one guard(schema_mgr=0x2b466bc04050, tenant_id=1, version=1734362366739016, cur_timestamp=1734400170266425, ref_timestamp=1734400110261460, lbt()="0x7bcec65 0x1a6c403f 0x78e2779 0x78e28f0 0x76151c5 0xd191344 0xbfe6d2f 0x1f623704 0x1f629f7c 0x1f629758 0x1f6278be 0x2b456c0fbea5 0x2b456c40e96d") [2024-12-16 17:49:30.267130] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:100) [128661][T1_REST_SER][T1][YB42C0A89F89-00062963E053613C-0-0] [lt=24] [RESTORE] restore scheduler start [2024-12-16 17:49:30.267150] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:107) [128661][T1_REST_SER][T1][YB42C0A89F89-00062963E053613D-0-0] [lt=19] [RESTORE] try process restore job [2024-12-16 17:49:30.269353] INFO [COMMON] get_jobs (ob_physical_restore_table_operator.cpp:419) [128661][T1_REST_SER][T1][YB42C0A89F89-00062963E053613D-0-0] [lt=6] [RESTORE] get restore jobs(ret=0, job_cnt=0) [2024-12-16 17:49:30.488227] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F411-0-0] [lt=51] [COMMON_LS_SERVICE] start to create ls(info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, create_scn={val:1734359202119679000, v:0}) [2024-12-16 17:49:30.504882] WDIAG [RS] do_create_user_ls (ob_common_ls_service.cpp:269) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F411-0-0] [lt=8][errcode=-4672] failed to create user ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, locality_array=[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], create_scn={val:1734359202119679000, v:0}, palf_base_info={prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}, curr_lsn:{lsn:18446744073709551615}}, create_with_palf=false, source_tenant_id=0) [2024-12-16 17:49:30.504930] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:274) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F411-0-0] [lt=38] [COMMON_LS_SERVICE] end to create ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, cost=16660) [2024-12-16 17:49:30.504943] WDIAG [RS] try_create_ls_ (ob_common_ls_service.cpp:179) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F411-0-0] [lt=12][errcode=-4672] failed to create new ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", status_info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, recovery_stat={tenant_id:1002, ls_id:{id:1003}, sync_scn:{val:1, v:0}, readable_scn:{val:1, v:0}, create_scn:{val:1734359202119679000, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:9223372036854775807, config_seq:-1}}) [2024-12-16 17:49:30.504962] WDIAG [RS] do_work (ob_common_ls_service.cpp:117) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F411-0-0] [lt=17][errcode=0] failed to create ls(ret=0, ret="OB_SUCCESS", tmp_ret=-4672, tmp_ret="OB_REPLICA_NUM_NOT_ENOUGH", user_tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:30.510136] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F411-0-0] [lt=114] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}, {unit_group_id:1002, unit_status:0, ls_group_ids:[]}]) [2024-12-16 17:49:30.512967] INFO [RS] do_work (ob_common_ls_service.cpp:138) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F411-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:30.549382] INFO [RS] do_work (ob_common_ls_service.cpp:138) [128656][T1_COMMONLSSe][T1][YB42C0A89F89-00062963E043FA79-0-0] [lt=29] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:30.733029] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1734400168945790000, replayable_scn:1734400168945790000, readable_scn:1734359204309426002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1734400170728307, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1734400168945790000, replayable_scn:1734400168945790000, readable_scn:1734359204309426002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1734400170728307, ora_rowscn_=1734400170093055000, content_changed=true) [2024-12-16 17:49:30.737574] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=24] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.159.137:2882") [2024-12-16 17:49:30.737598] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=23] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="192.168.159.138:2882") [2024-12-16 17:49:30.737607] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="192.168.159.139:2882") [2024-12-16 17:49:30.737680] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=72] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=3, dest="192.168.159.141:2882") [2024-12-16 17:49:30.737689] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=4, dest="192.168.159.142:2882") [2024-12-16 17:49:30.737696] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=5, dest="192.168.159.143:2882") [2024-12-16 17:49:30.773731] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129974][T1001_StandbySc][T1001][YB42C0A89F89-00062963E5D3F94F-0-0] [lt=31] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:30.819952] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128850][T1_L0_G0][T1][YB42C0A89F89-00062963E2039F02-0-0] [lt=43] [DDL] execute ddl like stmt(ret=0, cost=11, ddl_arg=NULL) [2024-12-16 17:49:30.963951] INFO [RS] process (ob_root_inspection.cpp:494) [128523][RSInspectTask][T0][Y0-00062963E494A262-0-0] [lt=89] purge recyclebin task begin [2024-12-16 17:49:30.963974] INFO [RS] process (ob_root_inspection.cpp:512) [128523][RSInspectTask][T0][Y0-00062963E494A262-0-0] [lt=20] submit purge recyclebin task success(delay=60000000) [2024-12-16 17:49:30.963982] INFO [RS] process (ob_root_inspection.cpp:514) [128523][RSInspectTask][T0][Y0-00062963E494A262-0-0] [lt=8] purge recyclebin task end(delay=60000000) [2024-12-16 17:49:31.024722] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129809][T1002_StandbySc][T1002][YB42C0A89F89-00062963E533F951-0-0] [lt=74] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:31.035922] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [128685][T1_HBService][T1][YB42C0A89F89-00062963E023AA0D-0-0] [lt=49] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:31.044507] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F89-00062963E103A9E3-0-0] [lt=68] [DDL] execute ddl like stmt(ret=0, cost=18, ddl_arg=NULL) [2024-12-16 17:49:31.112267] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [130096][T1001_FrzInfoDe][T1001][YB42C0A89F89-00062963E7F36925-0-0] [lt=30] get number(ret=0, service_type_=0, range=1, base_id=1734400171110075000, start_id=1734400171110075000, end_id=1734400171110075001) [2024-12-16 17:49:31.119782] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [130096][T1001_FrzInfoDe][T1001][YB42C0A89F89-00062963E7F36925-0-0] [lt=33] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1734400171110075000, v:0}, tenant_id=1001) [2024-12-16 17:49:31.121782] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8F-000629642BDC7AD2-0-0] [lt=29] [DDL] execute ddl like stmt(ret=0, cost=18, ddl_arg=NULL) [2024-12-16 17:49:31.141151] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F421-0-0] [lt=16] is primary tenant, no need check readable_scn [2024-12-16 17:49:31.152803] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F421-0-0] [lt=45] is primary tenant, no need check readable_scn [2024-12-16 17:49:31.158038] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [128655][T1_StandbySchem][T1][YB42C0A89F89-00062963DD93FAC9-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:31.161844] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F421-0-0] [lt=164] is primary tenant, no need check readable_scn [2024-12-16 17:49:31.196295] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F7FC-0-0] [lt=44] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202081794000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202110029000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:0, operation_type_:0, create_scn:{val:1734359202119679000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:31.199205] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F7FC-0-0] [lt=124] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-16 17:49:31.199231] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F7FC-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:31.311303] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8A-00062963DFD99B27-0-0] [lt=21] [DDL] execute ddl like stmt(ret=0, cost=59, ddl_arg=NULL) [2024-12-16 17:49:31.372853] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8D-000629641B98C908-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=19, ddl_arg=NULL) [2024-12-16 17:49:31.421471] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128853][T1_L0_G0][T1][YB42C0A89F8D-000629641D38A495-0-0] [lt=40] [DDL] execute ddl like stmt(ret=0, cost=11, ddl_arg=NULL) [2024-12-16 17:49:31.516443] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F412-0-0] [lt=39] [COMMON_LS_SERVICE] start to create ls(info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, create_scn={val:1734359202119679000, v:0}) [2024-12-16 17:49:31.530958] WDIAG [RS] do_create_user_ls (ob_common_ls_service.cpp:269) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F412-0-0] [lt=9][errcode=-4672] failed to create user ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, locality_array=[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], create_scn={val:1734359202119679000, v:0}, palf_base_info={prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}, curr_lsn:{lsn:18446744073709551615}}, create_with_palf=false, source_tenant_id=0) [2024-12-16 17:49:31.531004] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:274) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F412-0-0] [lt=37] [COMMON_LS_SERVICE] end to create ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, cost=14529) [2024-12-16 17:49:31.531017] WDIAG [RS] try_create_ls_ (ob_common_ls_service.cpp:179) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F412-0-0] [lt=11][errcode=-4672] failed to create new ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", status_info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, recovery_stat={tenant_id:1002, ls_id:{id:1003}, sync_scn:{val:1, v:0}, readable_scn:{val:1, v:0}, create_scn:{val:1734359202119679000, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:9223372036854775807, config_seq:-1}}) [2024-12-16 17:49:31.531037] WDIAG [RS] do_work (ob_common_ls_service.cpp:117) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F412-0-0] [lt=18][errcode=0] failed to create ls(ret=0, ret="OB_SUCCESS", tmp_ret=-4672, tmp_ret="OB_REPLICA_NUM_NOT_ENOUGH", user_tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:31.536065] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F412-0-0] [lt=41] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}, {unit_group_id:1002, unit_status:0, ls_group_ids:[]}]) [2024-12-16 17:49:31.543503] INFO [RS] do_work (ob_common_ls_service.cpp:138) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F412-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:31.551455] INFO [RS] do_work (ob_common_ls_service.cpp:138) [128656][T1_COMMONLSSe][T1][YB42C0A89F89-00062963E043FA7A-0-0] [lt=43] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:31.712815] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [128684][T1_HBService][T1][YB42C0A89F89-00062963E003A9C1-0-0] [lt=105] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:31.717330] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [129880][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=86] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1734400170728307, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1734400168945790000, replayable_scn:1734400168945790000, readable_scn:1734359204309426002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-16 17:49:31.758777] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.159.137:2882") [2024-12-16 17:49:31.758799] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=22] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="192.168.159.138:2882") [2024-12-16 17:49:31.758808] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="192.168.159.139:2882") [2024-12-16 17:49:31.758816] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=3, dest="192.168.159.141:2882") [2024-12-16 17:49:31.758823] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=4, dest="192.168.159.142:2882") [2024-12-16 17:49:31.758829] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=5, dest="192.168.159.143:2882") [2024-12-16 17:49:31.760975] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-16 17:49:31.773909] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129974][T1001_StandbySc][T1001][YB42C0A89F89-00062963E5D3F950-0-0] [lt=81] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:31.903535] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8B-00062963DFDFA01D-0-0] [lt=29] [DDL] execute ddl like stmt(ret=0, cost=19, ddl_arg=NULL) [2024-12-16 17:49:32.024974] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129809][T1002_StandbySc][T1002][YB42C0A89F89-00062963E533F952-0-0] [lt=152] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:32.027909] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8E-0006296420107352-0-0] [lt=32] [DDL] execute ddl like stmt(ret=0, cost=68, ddl_arg=NULL) [2024-12-16 17:49:32.158140] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [128655][T1_StandbySchem][T1][YB42C0A89F89-00062963DD93FACA-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:32.175456] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F422-0-0] [lt=16] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.159.137:2882", readable_scn:{val:1734400171951379000, v:0}}, {server:"192.168.159.138:2882", readable_scn:{val:1734400171951379000, v:0}}, {server:"192.168.159.139:2882", readable_scn:{val:1734400171951379000, v:0}}], last_dump_ts_=1734400166942038, force_dump=false) [2024-12-16 17:49:32.175514] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F422-0-0] [lt=50] is primary tenant, no need check readable_scn [2024-12-16 17:49:32.187307] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F422-0-0] [lt=11] is primary tenant, no need check readable_scn [2024-12-16 17:49:32.195080] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F422-0-0] [lt=35] is primary tenant, no need check readable_scn [2024-12-16 17:49:32.204570] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F7FD-0-0] [lt=114] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202081794000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202110029000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:0, operation_type_:0, create_scn:{val:1734359202119679000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:32.208082] INFO [SHARE] load_all_compaction_scn (ob_tablet_checksum_operator.cpp:638) [128839][T1_MFLaunch][T1][Y0-0000000000000000-0-0] [lt=36] finish to load all compaction_scn(ret=0, ret="OB_SUCCESS", tenant_id=1, cost_time_us=3359, estimated_timeout_us=9000000, sql=SELECT DISTINCT compaction_scn as dis_compaction_scn FROM __all_tablet_checksum WHERE tenant_id = 0 ORDER BY compaction_scn ASC, compaction_scn_arr=[]) [2024-12-16 17:49:32.208260] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F7FD-0-0] [lt=184] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-16 17:49:32.208280] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F7FD-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:32.482850] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128850][T1_L0_G0][T1][YB42C0A89F8B-00062963E0AF7A9E-0-0] [lt=66] [DDL] execute ddl like stmt(ret=0, cost=12, ddl_arg=NULL) [2024-12-16 17:49:32.547454] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F413-0-0] [lt=47] [COMMON_LS_SERVICE] start to create ls(info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, create_scn={val:1734359202119679000, v:0}) [2024-12-16 17:49:32.562753] INFO [RS] do_work (ob_common_ls_service.cpp:138) [128656][T1_COMMONLSSe][T1][YB42C0A89F89-00062963E043FA7B-0-0] [lt=45] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:32.578310] WDIAG [RS] do_create_user_ls (ob_common_ls_service.cpp:269) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F413-0-0] [lt=26][errcode=-4672] failed to create user ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, locality_array=[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], create_scn={val:1734359202119679000, v:0}, palf_base_info={prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}, curr_lsn:{lsn:18446744073709551615}}, create_with_palf=false, source_tenant_id=0) [2024-12-16 17:49:32.578360] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:274) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F413-0-0] [lt=40] [COMMON_LS_SERVICE] end to create ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, cost=30874) [2024-12-16 17:49:32.578373] WDIAG [RS] try_create_ls_ (ob_common_ls_service.cpp:179) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F413-0-0] [lt=12][errcode=-4672] failed to create new ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", status_info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, recovery_stat={tenant_id:1002, ls_id:{id:1003}, sync_scn:{val:1, v:0}, readable_scn:{val:1, v:0}, create_scn:{val:1734359202119679000, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:9223372036854775807, config_seq:-1}}) [2024-12-16 17:49:32.578394] WDIAG [RS] do_work (ob_common_ls_service.cpp:117) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F413-0-0] [lt=18][errcode=0] failed to create ls(ret=0, ret="OB_SUCCESS", tmp_ret=-4672, tmp_ret="OB_REPLICA_NUM_NOT_ENOUGH", user_tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:32.584205] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F413-0-0] [lt=43] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}, {unit_group_id:1002, unit_status:0, ls_group_ids:[]}]) [2024-12-16 17:49:32.586864] INFO [RS] do_work (ob_common_ls_service.cpp:138) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F413-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:32.737722] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.159.137:2882") [2024-12-16 17:49:32.737762] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=39] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="192.168.159.138:2882") [2024-12-16 17:49:32.737774] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="192.168.159.139:2882") [2024-12-16 17:49:32.737782] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=3, dest="192.168.159.141:2882") [2024-12-16 17:49:32.737789] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=4, dest="192.168.159.142:2882") [2024-12-16 17:49:32.737795] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=5, dest="192.168.159.143:2882") [2024-12-16 17:49:32.737808] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=2577, return_code_array=[0, 0, 0, 0, 0, 0], broadcast_times=1) [2024-12-16 17:49:32.774093] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129974][T1001_StandbySc][T1001][YB42C0A89F89-00062963E5D3F951-0-0] [lt=21] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:32.851281] INFO [RS] get_dbms_sched_job_infos_in_tenant (ob_dbms_sched_table_operator.cpp:677) [130065][T1001_DBMSSched][T1001][Y0-0000000000000000-0-0] [lt=98] failed to get result(ret=-4008) [2024-12-16 17:49:33.025192] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129809][T1002_StandbySc][T1002][YB42C0A89F89-00062963E533F953-0-0] [lt=46] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:33.042134] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [128685][T1_HBService][T1][YB42C0A89F89-00062963E023AA0E-0-0] [lt=26] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:33.046233] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F89-00062963E103A9E4-0-0] [lt=27] [DDL] execute ddl like stmt(ret=0, cost=132, ddl_arg=NULL) [2024-12-16 17:49:33.095513] INFO [RS] get_dbms_sched_job_infos_in_tenant (ob_dbms_sched_table_operator.cpp:677) [128751][T1_DBMSSched][T1][Y0-0000000000000000-0-0] [lt=26] failed to get result(ret=-4008) [2024-12-16 17:49:33.126550] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8F-000629642BDC7AD3-0-0] [lt=39] [DDL] execute ddl like stmt(ret=0, cost=18, ddl_arg=NULL) [2024-12-16 17:49:33.158248] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [128655][T1_StandbySchem][T1][YB42C0A89F89-00062963DD93FACB-0-0] [lt=23] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:33.212780] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F7FE-0-0] [lt=47] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202081794000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202110029000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:0, operation_type_:0, create_scn:{val:1734359202119679000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:33.216162] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F7FE-0-0] [lt=99] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-16 17:49:33.216182] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F7FE-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:33.220163] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F423-0-0] [lt=34] is primary tenant, no need check readable_scn [2024-12-16 17:49:33.240987] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F423-0-0] [lt=19] is primary tenant, no need check readable_scn [2024-12-16 17:49:33.255303] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F423-0-0] [lt=21] is primary tenant, no need check readable_scn [2024-12-16 17:49:33.255365] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:464) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F423-0-0] [lt=20] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1734400171074741000, v:0}) [2024-12-16 17:49:33.314512] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8A-00062963DFD99B28-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=22, ddl_arg=NULL) [2024-12-16 17:49:33.373417] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8D-000629641B98C909-0-0] [lt=37] [DDL] execute ddl like stmt(ret=0, cost=29, ddl_arg=NULL) [2024-12-16 17:49:33.419240] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [128532][DBMS_JOB_MASTER][T0][YB42C0A89F89-00062963DE235926-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2024-12-16 17:49:33.420962] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [128532][DBMS_JOB_MASTER][T0][YB42C0A89F89-00062963DE235926-0-0] [lt=23] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1734400173416449, delay:5000000, job_id:0, tenant_id:1002}, job_key=0x2b45b422a0e0, job_infos=[]) [2024-12-16 17:49:33.420990] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [128532][DBMS_JOB_MASTER][T0][YB42C0A89F89-00062963DE235926-0-0] [lt=24] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1734400178420987, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-16 17:49:33.530891] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [128532][DBMS_JOB_MASTER][T0][YB42C0A89F89-00062963DE235926-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-16 17:49:33.532503] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [128532][DBMS_JOB_MASTER][T0][YB42C0A89F89-00062963DE235926-0-0] [lt=21] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1734400173530721, delay:5000000, job_id:0, tenant_id:1001}, job_key=0x2b45b422a0a0, job_infos=[]) [2024-12-16 17:49:33.532531] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [128532][DBMS_JOB_MASTER][T0][YB42C0A89F89-00062963DE235926-0-0] [lt=26] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1734400178532528, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-16 17:49:33.532548] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [128532][DBMS_JOB_MASTER][T0][YB42C0A89F89-00062963DE235926-0-0] [lt=12] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1734400178532544, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-16 17:49:33.565394] INFO [RS] do_work (ob_common_ls_service.cpp:138) [128656][T1_COMMONLSSe][T1][YB42C0A89F89-00062963E043FA7C-0-0] [lt=43] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:33.590886] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F414-0-0] [lt=19] [COMMON_LS_SERVICE] start to create ls(info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, create_scn={val:1734359202119679000, v:0}) [2024-12-16 17:49:33.606465] WDIAG [RS] do_create_user_ls (ob_common_ls_service.cpp:269) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F414-0-0] [lt=7][errcode=-4672] failed to create user ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, locality_array=[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], create_scn={val:1734359202119679000, v:0}, palf_base_info={prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}, curr_lsn:{lsn:18446744073709551615}}, create_with_palf=false, source_tenant_id=0) [2024-12-16 17:49:33.606548] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:274) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F414-0-0] [lt=71] [COMMON_LS_SERVICE] end to create ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, cost=15600) [2024-12-16 17:49:33.606564] WDIAG [RS] try_create_ls_ (ob_common_ls_service.cpp:179) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F414-0-0] [lt=16][errcode=-4672] failed to create new ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", status_info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, recovery_stat={tenant_id:1002, ls_id:{id:1003}, sync_scn:{val:1, v:0}, readable_scn:{val:1, v:0}, create_scn:{val:1734359202119679000, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:9223372036854775807, config_seq:-1}}) [2024-12-16 17:49:33.606585] WDIAG [RS] do_work (ob_common_ls_service.cpp:117) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F414-0-0] [lt=18][errcode=0] failed to create ls(ret=0, ret="OB_SUCCESS", tmp_ret=-4672, tmp_ret="OB_REPLICA_NUM_NOT_ENOUGH", user_tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:33.612155] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F414-0-0] [lt=42] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}, {unit_group_id:1002, unit_status:0, ls_group_ids:[]}]) [2024-12-16 17:49:33.614991] INFO [RS] do_work (ob_common_ls_service.cpp:138) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F414-0-0] [lt=69] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:33.716107] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [128684][T1_HBService][T1][YB42C0A89F89-00062963E003A9C2-0-0] [lt=18] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:33.736798] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1734400171951379000, replayable_scn:1734400171951379000, readable_scn:1734359204309426002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1734400173731248, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1734400171951379000, replayable_scn:1734400171951379000, readable_scn:1734359204309426002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1734400173731248, ora_rowscn_=1734400173223617000, content_changed=true) [2024-12-16 17:49:33.738998] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=26] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.159.137:2882") [2024-12-16 17:49:33.739039] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=40] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="192.168.159.138:2882") [2024-12-16 17:49:33.739050] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="192.168.159.139:2882") [2024-12-16 17:49:33.739057] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=3, dest="192.168.159.141:2882") [2024-12-16 17:49:33.739064] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=4, dest="192.168.159.142:2882") [2024-12-16 17:49:33.739071] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=5, dest="192.168.159.143:2882") [2024-12-16 17:49:33.774286] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129974][T1001_StandbySc][T1001][YB42C0A89F89-00062963E5D3F952-0-0] [lt=56] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:33.802829] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [128834][T1_FrzInfoDet][T1][YB42C0A89F89-00062963DF93694A-0-0] [lt=139] get number(ret=0, service_type_=0, range=1, base_id=1734400173800959000, start_id=1734400173800959000, end_id=1734400173800959001) [2024-12-16 17:49:33.820134] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [128834][T1_FrzInfoDet][T1][YB42C0A89F89-00062963DF93694A-0-0] [lt=42] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1734400173800959000, v:0}, tenant_id=1) [2024-12-16 17:49:33.905859] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8B-00062963DFDFA01E-0-0] [lt=29] [DDL] execute ddl like stmt(ret=0, cost=22, ddl_arg=NULL) [2024-12-16 17:49:34.026930] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129809][T1002_StandbySc][T1002][YB42C0A89F89-00062963E533F954-0-0] [lt=31] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:34.030773] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8E-0006296420107353-0-0] [lt=36] [DDL] execute ddl like stmt(ret=0, cost=18, ddl_arg=NULL) [2024-12-16 17:49:34.158613] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [128655][T1_StandbySchem][T1][YB42C0A89F89-00062963DD93FACC-0-0] [lt=43] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:34.221790] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F7FF-0-0] [lt=41] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202081794000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202110029000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:0, operation_type_:0, create_scn:{val:1734359202119679000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:34.225410] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F7FF-0-0] [lt=145] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-16 17:49:34.225523] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F7FF-0-0] [lt=112] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:34.233520] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128853][T1_L0_G0][T1][YB42C0A89F8F-000629642D9C5718-0-0] [lt=39] [DDL] execute ddl like stmt(ret=0, cost=45, ddl_arg=NULL) [2024-12-16 17:49:34.270447] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F424-0-0] [lt=14] is primary tenant, no need check readable_scn [2024-12-16 17:49:34.289834] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F424-0-0] [lt=19] is primary tenant, no need check readable_scn [2024-12-16 17:49:34.305481] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F424-0-0] [lt=23] is primary tenant, no need check readable_scn [2024-12-16 17:49:34.569597] INFO [RS] do_work (ob_common_ls_service.cpp:138) [128656][T1_COMMONLSSe][T1][YB42C0A89F89-00062963E043FA7D-0-0] [lt=59] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:34.619725] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F415-0-0] [lt=40] [COMMON_LS_SERVICE] start to create ls(info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, create_scn={val:1734359202119679000, v:0}) [2024-12-16 17:49:34.635506] WDIAG [RS] do_create_user_ls (ob_common_ls_service.cpp:269) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F415-0-0] [lt=11][errcode=-4672] failed to create user ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, locality_array=[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], create_scn={val:1734359202119679000, v:0}, palf_base_info={prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}, curr_lsn:{lsn:18446744073709551615}}, create_with_palf=false, source_tenant_id=0) [2024-12-16 17:49:34.635557] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:274) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F415-0-0] [lt=38] [COMMON_LS_SERVICE] end to create ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, cost=15790) [2024-12-16 17:49:34.635570] WDIAG [RS] try_create_ls_ (ob_common_ls_service.cpp:179) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F415-0-0] [lt=12][errcode=-4672] failed to create new ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", status_info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, recovery_stat={tenant_id:1002, ls_id:{id:1003}, sync_scn:{val:1, v:0}, readable_scn:{val:1, v:0}, create_scn:{val:1734359202119679000, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:9223372036854775807, config_seq:-1}}) [2024-12-16 17:49:34.635699] WDIAG [RS] do_work (ob_common_ls_service.cpp:117) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F415-0-0] [lt=35][errcode=0] failed to create ls(ret=0, ret="OB_SUCCESS", tmp_ret=-4672, tmp_ret="OB_REPLICA_NUM_NOT_ENOUGH", user_tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:34.641235] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F415-0-0] [lt=51] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}, {unit_group_id:1002, unit_status:0, ls_group_ids:[]}]) [2024-12-16 17:49:34.644141] INFO [RS] do_work (ob_common_ls_service.cpp:138) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F415-0-0] [lt=42] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:34.715551] INFO [RS] recover_task (ob_ddl_scheduler.cpp:2516) [128993][DDLScanTask][T0][Y0-0000000000000000-0-0] [lt=17] start processing ddl recovery(ddl_event_info={addr_:"192.168.159.137:2882", event_ts_:1734400174715549, sub_id_:0, trace_id_:Y0-0000000000000000-0-0, parent_trace_id_:Y0-0000000000000000-0-0}, task_records=[], primary_tenant_ids=[1002]) [2024-12-16 17:49:34.732127] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [129880][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=17] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1734400173731248, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1734400171951379000, replayable_scn:1734400171951379000, readable_scn:1734359204309426002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-16 17:49:34.740130] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.159.137:2882") [2024-12-16 17:49:34.740177] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=45] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="192.168.159.138:2882") [2024-12-16 17:49:34.740189] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="192.168.159.139:2882") [2024-12-16 17:49:34.740197] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=3, dest="192.168.159.141:2882") [2024-12-16 17:49:34.740203] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=4, dest="192.168.159.142:2882") [2024-12-16 17:49:34.740210] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=5, dest="192.168.159.143:2882") [2024-12-16 17:49:34.774500] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129974][T1001_StandbySc][T1001][YB42C0A89F89-00062963E5D3F953-0-0] [lt=45] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:35.027104] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129809][T1002_StandbySc][T1002][YB42C0A89F89-00062963E533F955-0-0] [lt=59] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:35.048550] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [128685][T1_HBService][T1][YB42C0A89F89-00062963E023AA0F-0-0] [lt=53] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:35.049450] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F89-00062963E103A9E5-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=16, ddl_arg=NULL) [2024-12-16 17:49:35.136940] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8F-000629642BDC7AD4-0-0] [lt=22] [DDL] execute ddl like stmt(ret=0, cost=19, ddl_arg=NULL) [2024-12-16 17:49:35.159158] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [128655][T1_StandbySchem][T1][YB42C0A89F89-00062963DD93FACD-0-0] [lt=252] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:35.219605] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [130219][T1002_FrzInfoDe][T1002][YB42C0A89F89-00062963EAD36922-0-0] [lt=36] get number(ret=0, service_type_=0, range=1, base_id=1734400175216387000, start_id=1734400175216387000, end_id=1734400175216387001) [2024-12-16 17:49:35.228761] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F800-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202081794000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202110029000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:0, operation_type_:0, create_scn:{val:1734359202119679000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:35.231974] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F800-0-0] [lt=130] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-16 17:49:35.232014] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F800-0-0] [lt=39] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:35.235266] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [130219][T1002_FrzInfoDe][T1002][YB42C0A89F89-00062963EAD36922-0-0] [lt=83] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1734400175216387000, v:0}, tenant_id=1002) [2024-12-16 17:49:35.315569] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F425-0-0] [lt=20] is primary tenant, no need check readable_scn [2024-12-16 17:49:35.317232] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8A-00062963DFD99B29-0-0] [lt=181] [DDL] execute ddl like stmt(ret=0, cost=19, ddl_arg=NULL) [2024-12-16 17:49:35.324985] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:345) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F425-0-0] [lt=35] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1002, ls_id:{id:1}, sync_scn:{val:1734400175216387001, v:0}, readable_scn:{val:1734400175216387001, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:1, config_seq:2}}) [2024-12-16 17:49:35.326398] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128850][T1_L0_G0][T1][YB42C0A89F8A-00062963E0A97581-0-0] [lt=39] [DDL] execute ddl like stmt(ret=0, cost=11, ddl_arg=NULL) [2024-12-16 17:49:35.328910] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F425-0-0] [lt=52] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.159.137:2882", readable_scn:{val:1734400175234534000, v:0}}, {server:"192.168.159.138:2882", readable_scn:{val:1734400175234534000, v:0}}, {server:"192.168.159.139:2882", readable_scn:{val:1734400175234534000, v:0}}], last_dump_ts_=1734400170110461, force_dump=false) [2024-12-16 17:49:35.328949] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F425-0-0] [lt=33] is primary tenant, no need check readable_scn [2024-12-16 17:49:35.338426] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F425-0-0] [lt=8] is primary tenant, no need check readable_scn [2024-12-16 17:49:35.375138] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8D-000629641B98C90A-0-0] [lt=22] [DDL] execute ddl like stmt(ret=0, cost=25, ddl_arg=NULL) [2024-12-16 17:49:35.572352] INFO [RS] do_work (ob_common_ls_service.cpp:138) [128656][T1_COMMONLSSe][T1][YB42C0A89F89-00062963E043FA7E-0-0] [lt=33] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:35.651180] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F416-0-0] [lt=45] [COMMON_LS_SERVICE] start to create ls(info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, create_scn={val:1734359202119679000, v:0}) [2024-12-16 17:49:35.682422] WDIAG [RS] do_create_user_ls (ob_common_ls_service.cpp:269) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F416-0-0] [lt=12][errcode=-4672] failed to create user ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, locality_array=[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], create_scn={val:1734359202119679000, v:0}, palf_base_info={prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}, curr_lsn:{lsn:18446744073709551615}}, create_with_palf=false, source_tenant_id=0) [2024-12-16 17:49:35.682472] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:274) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F416-0-0] [lt=39] [COMMON_LS_SERVICE] end to create ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, cost=31232) [2024-12-16 17:49:35.682486] WDIAG [RS] try_create_ls_ (ob_common_ls_service.cpp:179) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F416-0-0] [lt=13][errcode=-4672] failed to create new ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", status_info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, recovery_stat={tenant_id:1002, ls_id:{id:1003}, sync_scn:{val:1, v:0}, readable_scn:{val:1, v:0}, create_scn:{val:1734359202119679000, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:9223372036854775807, config_seq:-1}}) [2024-12-16 17:49:35.682506] WDIAG [RS] do_work (ob_common_ls_service.cpp:117) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F416-0-0] [lt=18][errcode=0] failed to create ls(ret=0, ret="OB_SUCCESS", tmp_ret=-4672, tmp_ret="OB_REPLICA_NUM_NOT_ENOUGH", user_tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:35.689301] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F416-0-0] [lt=169] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}, {unit_group_id:1002, unit_status:0, ls_group_ids:[]}]) [2024-12-16 17:49:35.693369] INFO [RS] do_work (ob_common_ls_service.cpp:138) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F416-0-0] [lt=30] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:35.722152] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [128684][T1_HBService][T1][YB42C0A89F89-00062963E003A9C3-0-0] [lt=43] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:35.744201] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.159.137:2882") [2024-12-16 17:49:35.744248] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=46] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="192.168.159.138:2882") [2024-12-16 17:49:35.744259] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="192.168.159.139:2882") [2024-12-16 17:49:35.744267] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=3, dest="192.168.159.141:2882") [2024-12-16 17:49:35.744284] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=4, dest="192.168.159.142:2882") [2024-12-16 17:49:35.744291] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=5, dest="192.168.159.143:2882") [2024-12-16 17:49:35.744305] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=8246, return_code_array=[0, 0, 0, 0, 0, 0], broadcast_times=1) [2024-12-16 17:49:35.776584] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129974][T1001_StandbySc][T1001][YB42C0A89F89-00062963E5D3F954-0-0] [lt=52] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:35.909607] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8B-00062963DFDFA01F-0-0] [lt=63] [DDL] execute ddl like stmt(ret=0, cost=22, ddl_arg=NULL) [2024-12-16 17:49:36.027731] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129809][T1002_StandbySc][T1002][YB42C0A89F89-00062963E533F956-0-0] [lt=29] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:36.034059] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8E-0006296420107354-0-0] [lt=125] [DDL] execute ddl like stmt(ret=0, cost=20, ddl_arg=NULL) [2024-12-16 17:49:36.159319] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [128655][T1_StandbySchem][T1][YB42C0A89F89-00062963DD93FACE-0-0] [lt=21] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:36.237420] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F801-0-0] [lt=76] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202081794000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202110029000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:0, operation_type_:0, create_scn:{val:1734359202119679000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:36.242534] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F801-0-0] [lt=349] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-16 17:49:36.242565] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F801-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:36.353844] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F426-0-0] [lt=122] is primary tenant, no need check readable_scn [2024-12-16 17:49:36.370140] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F426-0-0] [lt=23] is primary tenant, no need check readable_scn [2024-12-16 17:49:36.377137] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F426-0-0] [lt=23] is primary tenant, no need check readable_scn [2024-12-16 17:49:36.575069] INFO [RS] do_work (ob_common_ls_service.cpp:138) [128656][T1_COMMONLSSe][T1][YB42C0A89F89-00062963E043FA7F-0-0] [lt=44] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:36.696792] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F417-0-0] [lt=30] [COMMON_LS_SERVICE] start to create ls(info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, create_scn={val:1734359202119679000, v:0}) [2024-12-16 17:49:36.712731] WDIAG [RS] do_create_user_ls (ob_common_ls_service.cpp:269) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F417-0-0] [lt=11][errcode=-4672] failed to create user ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, locality_array=[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], create_scn={val:1734359202119679000, v:0}, palf_base_info={prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}, curr_lsn:{lsn:18446744073709551615}}, create_with_palf=false, source_tenant_id=0) [2024-12-16 17:49:36.712780] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:274) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F417-0-0] [lt=39] [COMMON_LS_SERVICE] end to create ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, cost=15942) [2024-12-16 17:49:36.712793] WDIAG [RS] try_create_ls_ (ob_common_ls_service.cpp:179) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F417-0-0] [lt=12][errcode=-4672] failed to create new ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", status_info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, recovery_stat={tenant_id:1002, ls_id:{id:1003}, sync_scn:{val:1, v:0}, readable_scn:{val:1, v:0}, create_scn:{val:1734359202119679000, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:9223372036854775807, config_seq:-1}}) [2024-12-16 17:49:36.712814] WDIAG [RS] do_work (ob_common_ls_service.cpp:117) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F417-0-0] [lt=18][errcode=0] failed to create ls(ret=0, ret="OB_SUCCESS", tmp_ret=-4672, tmp_ret="OB_REPLICA_NUM_NOT_ENOUGH", user_tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:36.717840] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F417-0-0] [lt=43] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}, {unit_group_id:1002, unit_status:0, ls_group_ids:[]}]) [2024-12-16 17:49:36.721408] INFO [RS] do_work (ob_common_ls_service.cpp:138) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F417-0-0] [lt=102] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:36.741801] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.159.137:2882") [2024-12-16 17:49:36.741826] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=25] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="192.168.159.138:2882") [2024-12-16 17:49:36.741877] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=50] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="192.168.159.139:2882") [2024-12-16 17:49:36.741888] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=3, dest="192.168.159.141:2882") [2024-12-16 17:49:36.741895] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=4, dest="192.168.159.142:2882") [2024-12-16 17:49:36.741902] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=5, dest="192.168.159.143:2882") [2024-12-16 17:49:36.776877] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129974][T1001_StandbySc][T1001][YB42C0A89F89-00062963E5D3F955-0-0] [lt=152] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:37.017272] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:452) [129841][T1002_TntShared][T1002][Y0-0000000000000000-0-0] [lt=31] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2024-12-16 17:49:37.028038] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129809][T1002_StandbySc][T1002][YB42C0A89F89-00062963E533F957-0-0] [lt=37] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:37.053996] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [128685][T1_HBService][T1][YB42C0A89F89-00062963E023AA10-0-0] [lt=44] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:37.055036] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F89-00062963E103A9E6-0-0] [lt=28] [DDL] execute ddl like stmt(ret=0, cost=16, ddl_arg=NULL) [2024-12-16 17:49:37.144564] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8F-000629642BDC7AD5-0-0] [lt=22] [DDL] execute ddl like stmt(ret=0, cost=19, ddl_arg=NULL) [2024-12-16 17:49:37.159731] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [128655][T1_StandbySchem][T1][YB42C0A89F89-00062963DD93FACF-0-0] [lt=28] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:37.255328] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F802-0-0] [lt=960] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202081794000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202110029000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:0, operation_type_:0, create_scn:{val:1734359202119679000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:37.258725] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F802-0-0] [lt=102] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-16 17:49:37.258744] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F802-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:37.319123] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8A-00062963DFD99B2A-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=22, ddl_arg=NULL) [2024-12-16 17:49:37.376454] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8D-000629641B98C90B-0-0] [lt=79] [DDL] execute ddl like stmt(ret=0, cost=57, ddl_arg=NULL) [2024-12-16 17:49:37.387599] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F427-0-0] [lt=17] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.159.137:2882", readable_scn:{val:1734400176963786002, v:0}}, {server:"192.168.159.138:2882", readable_scn:{val:1734400176963786002, v:0}}, {server:"192.168.159.139:2882", readable_scn:{val:1734400176963786002, v:0}}], last_dump_ts_=1734400172175454, force_dump=false) [2024-12-16 17:49:37.387705] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F427-0-0] [lt=98] is primary tenant, no need check readable_scn [2024-12-16 17:49:37.409281] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F427-0-0] [lt=9] is primary tenant, no need check readable_scn [2024-12-16 17:49:37.425124] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F427-0-0] [lt=22] is primary tenant, no need check readable_scn [2024-12-16 17:49:37.577548] INFO [RS] do_work (ob_common_ls_service.cpp:138) [128656][T1_COMMONLSSe][T1][YB42C0A89F89-00062963E043FA80-0-0] [lt=28] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:37.725030] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F418-0-0] [lt=22] [COMMON_LS_SERVICE] start to create ls(info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, create_scn={val:1734359202119679000, v:0}) [2024-12-16 17:49:37.736412] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1734400176268495000, replayable_scn:1734400176268495000, readable_scn:1734359204309426002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1734400177733147, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1734400176268495000, replayable_scn:1734400176268495000, readable_scn:1734359204309426002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1734400177733147, ora_rowscn_=1734400177389550001, content_changed=true) [2024-12-16 17:49:37.742763] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=27] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.159.137:2882") [2024-12-16 17:49:37.742813] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=49] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="192.168.159.138:2882") [2024-12-16 17:49:37.742825] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="192.168.159.139:2882") [2024-12-16 17:49:37.742833] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=3, dest="192.168.159.141:2882") [2024-12-16 17:49:37.742840] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=4, dest="192.168.159.142:2882") [2024-12-16 17:49:37.742847] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=5, dest="192.168.159.143:2882") [2024-12-16 17:49:37.744433] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-16 17:49:37.745122] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [128684][T1_HBService][T1][YB42C0A89F89-00062963E003A9C4-0-0] [lt=190] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:37.746021] WDIAG [RS] do_create_user_ls (ob_common_ls_service.cpp:269) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F418-0-0] [lt=8][errcode=-4672] failed to create user ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, locality_array=[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], create_scn={val:1734359202119679000, v:0}, palf_base_info={prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}, curr_lsn:{lsn:18446744073709551615}}, create_with_palf=false, source_tenant_id=0) [2024-12-16 17:49:37.746068] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:274) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F418-0-0] [lt=37] [COMMON_LS_SERVICE] end to create ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, cost=20990) [2024-12-16 17:49:37.746093] WDIAG [RS] try_create_ls_ (ob_common_ls_service.cpp:179) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F418-0-0] [lt=24][errcode=-4672] failed to create new ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", status_info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, recovery_stat={tenant_id:1002, ls_id:{id:1003}, sync_scn:{val:1, v:0}, readable_scn:{val:1, v:0}, create_scn:{val:1734359202119679000, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:9223372036854775807, config_seq:-1}}) [2024-12-16 17:49:37.746113] WDIAG [RS] do_work (ob_common_ls_service.cpp:117) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F418-0-0] [lt=17][errcode=0] failed to create ls(ret=0, ret="OB_SUCCESS", tmp_ret=-4672, tmp_ret="OB_REPLICA_NUM_NOT_ENOUGH", user_tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:37.751100] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F418-0-0] [lt=41] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}, {unit_group_id:1002, unit_status:0, ls_group_ids:[]}]) [2024-12-16 17:49:37.754304] INFO [RS] do_work (ob_common_ls_service.cpp:138) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F418-0-0] [lt=88] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:37.756013] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [129880][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=44] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1734400177733147, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1734400176268495000, replayable_scn:1734400176268495000, readable_scn:1734359204309426002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-16 17:49:37.777094] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129974][T1001_StandbySc][T1001][YB42C0A89F89-00062963E5D3F956-0-0] [lt=45] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:37.912195] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8B-00062963DFDFA020-0-0] [lt=29] [DDL] execute ddl like stmt(ret=0, cost=29, ddl_arg=NULL) [2024-12-16 17:49:38.028170] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129809][T1002_StandbySc][T1002][YB42C0A89F89-00062963E533F958-0-0] [lt=26] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:38.038030] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8E-0006296420107355-0-0] [lt=82] [DDL] execute ddl like stmt(ret=0, cost=44, ddl_arg=NULL) [2024-12-16 17:49:38.159883] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [128655][T1_StandbySchem][T1][YB42C0A89F89-00062963DD93FAD0-0-0] [lt=22] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:38.263602] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F803-0-0] [lt=42] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202081794000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202110029000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:0, operation_type_:0, create_scn:{val:1734359202119679000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:38.266809] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F803-0-0] [lt=312] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-16 17:49:38.266849] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F803-0-0] [lt=38] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:38.421171] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [128532][DBMS_JOB_MASTER][T0][YB42C0A89F89-00062963DE235926-0-0] [lt=13] dbms job master wait timeout, no entry(ret=-4018) [2024-12-16 17:49:38.424235] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [128532][DBMS_JOB_MASTER][T0][YB42C0A89F89-00062963DE235926-0-0] [lt=34] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1734400178420987, delay:5000000, job_id:0, tenant_id:1002}, job_key=0x2b45b422a0e0, job_infos=[]) [2024-12-16 17:49:38.424351] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [128532][DBMS_JOB_MASTER][T0][YB42C0A89F89-00062963DE235926-0-0] [lt=112] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1734400183424347, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-16 17:49:38.441439] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F428-0-0] [lt=18] is primary tenant, no need check readable_scn [2024-12-16 17:49:38.463493] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F428-0-0] [lt=22] is primary tenant, no need check readable_scn [2024-12-16 17:49:38.475831] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F428-0-0] [lt=36] is primary tenant, no need check readable_scn [2024-12-16 17:49:38.532702] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [128532][DBMS_JOB_MASTER][T0][YB42C0A89F89-00062963DE235926-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2024-12-16 17:49:38.534463] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [128532][DBMS_JOB_MASTER][T0][YB42C0A89F89-00062963DE235926-0-0] [lt=41] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1734400178532528, delay:5000000, job_id:0, tenant_id:1001}, job_key=0x2b45b422a0a0, job_infos=[]) [2024-12-16 17:49:38.534514] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [128532][DBMS_JOB_MASTER][T0][YB42C0A89F89-00062963DE235926-0-0] [lt=46] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1734400183534510, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-16 17:49:38.534534] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [128532][DBMS_JOB_MASTER][T0][YB42C0A89F89-00062963DE235926-0-0] [lt=15] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1734400183534530, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-16 17:49:38.581774] INFO [RS] do_work (ob_common_ls_service.cpp:138) [128656][T1_COMMONLSSe][T1][YB42C0A89F89-00062963E043FA81-0-0] [lt=42] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:38.739961] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.159.137:2882") [2024-12-16 17:49:38.739988] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=27] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="192.168.159.138:2882") [2024-12-16 17:49:38.739997] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="192.168.159.139:2882") [2024-12-16 17:49:38.740005] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=3, dest="192.168.159.141:2882") [2024-12-16 17:49:38.740012] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=4, dest="192.168.159.142:2882") [2024-12-16 17:49:38.740018] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=5, dest="192.168.159.143:2882") [2024-12-16 17:49:38.757343] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F419-0-0] [lt=19] [COMMON_LS_SERVICE] start to create ls(info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, create_scn={val:1734359202119679000, v:0}) [2024-12-16 17:49:38.771221] WDIAG [RS] do_create_user_ls (ob_common_ls_service.cpp:269) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F419-0-0] [lt=9][errcode=-4672] failed to create user ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, locality_array=[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], create_scn={val:1734359202119679000, v:0}, palf_base_info={prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}, curr_lsn:{lsn:18446744073709551615}}, create_with_palf=false, source_tenant_id=0) [2024-12-16 17:49:38.771269] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:274) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F419-0-0] [lt=37] [COMMON_LS_SERVICE] end to create ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, cost=13885) [2024-12-16 17:49:38.771281] WDIAG [RS] try_create_ls_ (ob_common_ls_service.cpp:179) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F419-0-0] [lt=11][errcode=-4672] failed to create new ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", status_info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, recovery_stat={tenant_id:1002, ls_id:{id:1003}, sync_scn:{val:1, v:0}, readable_scn:{val:1, v:0}, create_scn:{val:1734359202119679000, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:9223372036854775807, config_seq:-1}}) [2024-12-16 17:49:38.771302] WDIAG [RS] do_work (ob_common_ls_service.cpp:117) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F419-0-0] [lt=17][errcode=0] failed to create ls(ret=0, ret="OB_SUCCESS", tmp_ret=-4672, tmp_ret="OB_REPLICA_NUM_NOT_ENOUGH", user_tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:38.777388] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129974][T1001_StandbySc][T1001][YB42C0A89F89-00062963E5D3F957-0-0] [lt=22] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:38.778351] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F419-0-0] [lt=41] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}, {unit_group_id:1002, unit_status:0, ls_group_ids:[]}]) [2024-12-16 17:49:38.780922] INFO [RS] do_work (ob_common_ls_service.cpp:138) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F419-0-0] [lt=27] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:38.891282] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128850][T1_L0_G0][T1][YB42C0A89F8E-0006296421A04F41-0-0] [lt=41] [DDL] execute ddl like stmt(ret=0, cost=12, ddl_arg=NULL) [2024-12-16 17:49:39.028334] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129809][T1002_StandbySc][T1002][YB42C0A89F89-00062963E533F959-0-0] [lt=29] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:39.056374] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F89-00062963E103A9E7-0-0] [lt=27] [DDL] execute ddl like stmt(ret=0, cost=17, ddl_arg=NULL) [2024-12-16 17:49:39.060503] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [128685][T1_HBService][T1][YB42C0A89F89-00062963E023AA11-0-0] [lt=105] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:39.150739] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8F-000629642BDC7AD6-0-0] [lt=17] [DDL] execute ddl like stmt(ret=0, cost=97, ddl_arg=NULL) [2024-12-16 17:49:39.160703] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [128655][T1_StandbySchem][T1][YB42C0A89F89-00062963DD93FAD1-0-0] [lt=24] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:39.271072] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F804-0-0] [lt=46] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202081794000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202110029000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:0, operation_type_:0, create_scn:{val:1734359202119679000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:39.274031] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F804-0-0] [lt=132] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-16 17:49:39.274072] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F804-0-0] [lt=40] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:39.319928] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8A-00062963DFD99B2B-0-0] [lt=28] [DDL] execute ddl like stmt(ret=0, cost=75, ddl_arg=NULL) [2024-12-16 17:49:39.377142] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8D-000629641B98C90C-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=19, ddl_arg=NULL) [2024-12-16 17:49:39.485063] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F429-0-0] [lt=33] is primary tenant, no need check readable_scn [2024-12-16 17:49:39.496846] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F429-0-0] [lt=39] is primary tenant, no need check readable_scn [2024-12-16 17:49:39.503558] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F429-0-0] [lt=40] is primary tenant, no need check readable_scn [2024-12-16 17:49:39.583834] INFO [RS] do_work (ob_common_ls_service.cpp:138) [128656][T1_COMMONLSSe][T1][YB42C0A89F89-00062963E043FA82-0-0] [lt=138] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:39.666522] INFO [RS.LB] idle (ob_root_balancer.cpp:103) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D761-0-0] [lt=69] root balance idle(idle_time=10000000) [2024-12-16 17:49:39.666571] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2074) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=37] start try disaster recovery [2024-12-16 17:49:39.666586] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=12] start try tenant disaster recovery(tenant_id=1, only_for_display=false) [2024-12-16 17:49:39.668186] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=10] start try ls disaster recovery(ls_status_info={tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-16 17:49:39.675344] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=105] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1, ls_id={id:1}, acc_dr_task=0) [2024-12-16 17:49:39.675378] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=26] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:39.675391] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:1734358823275341, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.159.137:2882", sql_port:2881, role:1, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1734358823287139, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.159.138:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1734358823290920, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.159.139:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2024-12-16 17:49:39.675535] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=139] build ls locality stat map(tenant_id=1, ls_id={id:1}) [2024-12-16 17:49:39.675548] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-16 17:49:39.675561] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=13] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-16 17:49:39.675570] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-16 17:49:39.675581] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1734358823275341, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.159.137:2882", sql_port:2881, role:1, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1734358823287139, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.159.138:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1734358823290920, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.159.139:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-16 17:49:39.675862] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=223] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:39.675890] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=15] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:39.675901] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=9] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:39.675909] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:39.682187] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=7] try tenant disaster recovery succ(tenant_id=1, only_for_display=false) [2024-12-16 17:49:39.682214] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=25] start try tenant disaster recovery(tenant_id=1001, only_for_display=false) [2024-12-16 17:49:39.684046] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=10] start try ls disaster recovery(ls_status_info={tenant_id:1001, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-16 17:49:39.698380] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=26] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1001, ls_id={id:1}, acc_dr_task=0) [2024-12-16 17:49:39.698431] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=41] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:39.698447] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=12] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1001, ls_id:{id:1}, replicas:[{modify_time_us:1734359140223858, create_time_us:1734359134691166, tenant_id:1001, ls_id:{id:1}, server:"192.168.159.137:2882", sql_port:2881, role:1, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1734359134832984, create_time_us:1734359134670339, tenant_id:1001, ls_id:{id:1}, server:"192.168.159.138:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1734359135157794, create_time_us:1734359134696290, tenant_id:1001, ls_id:{id:1}, server:"192.168.159.139:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1001, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2024-12-16 17:49:39.698695] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=242] build ls locality stat map(tenant_id=1001, ls_id={id:1}) [2024-12-16 17:49:39.698714] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=13] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-16 17:49:39.698727] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=13] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-16 17:49:39.698737] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-16 17:49:39.698747] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1734359140223858, create_time_us:1734359134691166, tenant_id:1001, ls_id:{id:1}, server:"192.168.159.137:2882", sql_port:2881, role:1, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1734359134832984, create_time_us:1734359134670339, tenant_id:1001, ls_id:{id:1}, server:"192.168.159.138:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1734359135157794, create_time_us:1734359134696290, tenant_id:1001, ls_id:{id:1}, server:"192.168.159.139:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-16 17:49:39.698878] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=118] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:39.698896] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:39.698906] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:39.698915] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:39.700775] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=6] try tenant disaster recovery succ(tenant_id=1001, only_for_display=false) [2024-12-16 17:49:39.702420] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=39] start try tenant disaster recovery(tenant_id=1002, only_for_display=false) [2024-12-16 17:49:39.703844] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=17] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-16 17:49:39.712317] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=41] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1}, acc_dr_task=0) [2024-12-16 17:49:39.712377] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=50] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:39.712394] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=14] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1}, replicas:[{modify_time_us:1734359175153676, create_time_us:1734359170239483, tenant_id:1002, ls_id:{id:1}, server:"192.168.159.137:2882", sql_port:2881, role:1, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1734359171015694, create_time_us:1734359169618664, tenant_id:1002, ls_id:{id:1}, server:"192.168.159.138:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1734359170930591, create_time_us:1734359169573173, tenant_id:1002, ls_id:{id:1}, server:"192.168.159.139:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2024-12-16 17:49:39.712626] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=225] build ls locality stat map(tenant_id=1002, ls_id={id:1}) [2024-12-16 17:49:39.712648] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=16] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-16 17:49:39.712662] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=12] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-16 17:49:39.712671] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-16 17:49:39.712683] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1734359175153676, create_time_us:1734359170239483, tenant_id:1002, ls_id:{id:1}, server:"192.168.159.137:2882", sql_port:2881, role:1, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1734359171015694, create_time_us:1734359169618664, tenant_id:1002, ls_id:{id:1}, server:"192.168.159.138:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1734359170930591, create_time_us:1734359169573173, tenant_id:1002, ls_id:{id:1}, server:"192.168.159.139:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-16 17:49:39.712796] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=102] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:39.712814] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:39.712822] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:39.712829] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:39.716338] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-16 17:49:39.731201] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=44] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1001}, acc_dr_task=0) [2024-12-16 17:49:39.731235] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=26] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:39.731248] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1001}, replicas:[{modify_time_us:1734359215801871, create_time_us:1734359210352363, tenant_id:1002, ls_id:{id:1001}, server:"192.168.159.137:2882", sql_port:2881, role:1, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1734359211537200, create_time_us:1734359210337910, tenant_id:1002, ls_id:{id:1001}, server:"192.168.159.138:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1734359211479956, create_time_us:1734359210323237, tenant_id:1002, ls_id:{id:1001}, server:"192.168.159.139:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2024-12-16 17:49:39.731392] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=140] build ls locality stat map(tenant_id=1002, ls_id={id:1001}) [2024-12-16 17:49:39.731405] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-16 17:49:39.731417] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=12] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-16 17:49:39.731427] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-16 17:49:39.731438] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1734359215801871, create_time_us:1734359210352363, tenant_id:1002, ls_id:{id:1001}, server:"192.168.159.137:2882", sql_port:2881, role:1, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1734359211537200, create_time_us:1734359210337910, tenant_id:1002, ls_id:{id:1001}, server:"192.168.159.138:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1734359211479956, create_time_us:1734359210323237, tenant_id:1002, ls_id:{id:1001}, server:"192.168.159.139:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-16 17:49:39.731547] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=99] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:39.731563] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:39.731572] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:39.731580] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:39.733286] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-16 17:49:39.749017] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [128684][T1_HBService][T1][YB42C0A89F89-00062963E003A9C5-0-0] [lt=92] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:39.752377] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1002}, acc_dr_task=0) [2024-12-16 17:49:39.752412] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=26] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:39.752424] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1002}, replicas:[{modify_time_us:1734359211546178, create_time_us:1734359210734985, tenant_id:1002, ls_id:{id:1002}, server:"192.168.159.137:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1734359216339659, create_time_us:1734359210724652, tenant_id:1002, ls_id:{id:1002}, server:"192.168.159.138:2882", sql_port:2881, role:1, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1734359211505782, create_time_us:1734359210727004, tenant_id:1002, ls_id:{id:1002}, server:"192.168.159.139:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2024-12-16 17:49:39.752746] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.159.137:2882") [2024-12-16 17:49:39.752767] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=20] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="192.168.159.138:2882") [2024-12-16 17:49:39.752774] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="192.168.159.139:2882") [2024-12-16 17:49:39.752781] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=3, dest="192.168.159.141:2882") [2024-12-16 17:49:39.752787] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=4, dest="192.168.159.142:2882") [2024-12-16 17:49:39.752792] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=5, dest="192.168.159.143:2882") [2024-12-16 17:49:39.752803] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=12284, return_code_array=[0, 0, 0, 0, 0, 0], broadcast_times=2) [2024-12-16 17:49:39.755081] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=2648] build ls locality stat map(tenant_id=1002, ls_id={id:1002}) [2024-12-16 17:49:39.755122] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=34] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-16 17:49:39.755140] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=17] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-16 17:49:39.755149] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-16 17:49:39.755160] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1734359211546178, create_time_us:1734359210734985, tenant_id:1002, ls_id:{id:1002}, server:"192.168.159.137:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.159.137:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.137:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1734359216339659, create_time_us:1734359210724652, tenant_id:1002, ls_id:{id:1002}, server:"192.168.159.138:2882", sql_port:2881, role:1, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1002, in_pool:true, unit:unit_id:1002, resource_pool_id:1001, unit_group_id:1001, zone:"zone2", server:"192.168.159.138:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.138:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1734359211505782, create_time_us:1734359210727004, tenant_id:1002, ls_id:{id:1002}, server:"192.168.159.139:2882", sql_port:2881, role:2, member_list:[{server:"192.168.159.137:2882", timestamp:1}, {server:"192.168.159.138:2882", timestamp:1}, {server:"192.168.159.139:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1001, unit_group_id:1001, zone:"zone3", server:"192.168.159.139:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.159.139:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-16 17:49:39.755295] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=123] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:39.755330] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=11] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:39.755343] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=11] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:39.755351] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-16 17:49:39.757086] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-16 17:49:39.757111] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=21] try tenant disaster recovery succ(tenant_id=1002, only_for_display=false) [2024-12-16 17:49:39.757122] INFO [RS] check (ob_migrate_unit_finish_checker.cpp:79) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=8] start check unit migrate finish [2024-12-16 17:49:39.757135] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=6] try check migrate unit finish by tenant(tenant_id=1) [2024-12-16 17:49:39.763196] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=7] try check migrate unit finish by tenant(tenant_id=1002) [2024-12-16 17:49:39.777858] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129974][T1001_StandbySc][T1001][YB42C0A89F89-00062963E5D3F958-0-0] [lt=112] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:39.782475] WDIAG [SHARE.PT] find_leader (ob_ls_info.cpp:848) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=45][errcode=-4018] fail to get leader replica(ret=-4018, ret="OB_ENTRY_NOT_EXIST", *this={tenant_id:1002, ls_id:{id:1003}, replicas:[]}, replica count=0) [2024-12-16 17:49:39.782524] WDIAG [RS] construct_filtered_ls_info_to_use_ (ob_disaster_recovery_info.cpp:398) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=48][errcode=-4018] fail to find leader replica in input_ls_info(ret=-4018, ret="OB_ENTRY_NOT_EXIST", input_ls_info={tenant_id:1002, ls_id:{id:1003}, replicas:[]}) [2024-12-16 17:49:39.782539] WDIAG [RS] build_disaster_ls_info (ob_disaster_recovery_info.cpp:459) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=14][errcode=-4018] fail to filter not in member/learner list replicas and learner_with_flag replicas(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_info={tenant_id:1002, ls_id:{id:1003}, replicas:[]}, filter_readonly_replicas_with_flag=true) [2024-12-16 17:49:39.782550] WDIAG [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:200) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=10][errcode=-4018] fail to generate dr log stream info(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-16 17:49:39.783356] WDIAG [RS] check (ob_migrate_unit_finish_checker.cpp:96) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=7][errcode=0] fail to try check migrate unit finish by tenant(tmp_ret=-4018, tmp_ret="OB_ENTRY_NOT_EXIST", tenant_id=1002) [2024-12-16 17:49:39.783424] INFO [RS] check (ob_alter_locality_finish_checker.cpp:79) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D762-0-0] [lt=35] start to check alter locality finish [2024-12-16 17:49:39.784484] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D763-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2024-12-16 17:49:39.785587] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D763-0-0] [lt=16][errcode=-4008] empty result set(ret=-4008) [2024-12-16 17:49:39.789468] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41A-0-0] [lt=20] [COMMON_LS_SERVICE] start to create ls(info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, create_scn={val:1734359202119679000, v:0}) [2024-12-16 17:49:39.799031] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D765-0-0] [lt=102][errcode=-4008] empty result set(ret=-4008) [2024-12-16 17:49:39.800493] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D765-0-0] [lt=49][errcode=-4008] empty result set(ret=-4008) [2024-12-16 17:49:39.801829] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:72) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D765-0-0] [lt=22] start check shrink resource pool [2024-12-16 17:49:39.801851] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D766-0-0] [lt=15] start check shrink resource pool(tenant_id=1) [2024-12-16 17:49:39.803185] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D766-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2024-12-16 17:49:39.804432] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D766-0-0] [lt=61][errcode=-4008] empty result set(ret=-4008) [2024-12-16 17:49:39.804506] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D767-0-0] [lt=15] start check shrink resource pool(tenant_id=1002) [2024-12-16 17:49:39.808329] INFO [SHARE] get_balance_job (ob_balance_job_table_operator.cpp:209) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D767-0-0] [lt=86] empty balance job(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sql=select time_to_usec(gmt_create) as start_time, time_to_usec(gmt_modified) as finish_time, * from __all_balance_job) [2024-12-16 17:49:39.812317] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128852][T1_L0_G0][T1][YB42C0A89F89-00062963E2039F06-0-0] [lt=20] [DDL] execute ddl like stmt(ret=0, cost=11, ddl_arg=NULL) [2024-12-16 17:49:39.817901] WDIAG [BALANCE] gather_ls_status_stat (ob_tenant_balance_service.cpp:264) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D767-0-0] [lt=19][errcode=-4076] [TENANT_BALANCE] ls status not ready, can not balance(ret=-4076, ret="OB_NEED_WAIT", ls_info={id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:0, operation_type_:0, create_scn:{val:1734359202119679000, v:0}}, status_info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-16 17:49:39.817956] WDIAG [BALANCE] is_primary_tenant_ls_balance_finished_ (ob_tenant_balance_service.cpp:366) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D767-0-0] [lt=51][errcode=-4076] fail to execute gather_ls_status_stat(ret=-4076, ret="OB_NEED_WAIT", tenant_id=1002) [2024-12-16 17:49:39.817969] INFO [BALANCE] is_primary_tenant_ls_balance_finished_ (ob_tenant_balance_service.cpp:379) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D767-0-0] [lt=12] check whether the primary_tenant has balanced ls(ret=-4076, ret="OB_NEED_WAIT", tenant_id=1002, ls_array=[{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, {tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, {tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, {tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}], primary_zone_num=0, unit_group_array=[], need_ls_balance=false) [2024-12-16 17:49:39.818022] WDIAG begin (ob_hashtable.h:932) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D767-0-0] [lt=42][errcode=-4006] hashtable not init, backtrace=0x7bcec65 0xe3d6410 0xe3c663b 0xce43adb 0xce40eb5 0xc693a3d 0xc822979 0xcf8b6dc 0xc7e823a 0xc7e4fbd 0xc7e49a4 0x1f623704 0x1f629f7c 0x1f629758 0x1f6278f7 0x2b456c0fbea5 0x2b456c40e96d [2024-12-16 17:49:39.818031] WDIAG [BALANCE] is_ls_balance_finished (ob_tenant_balance_service.cpp:315) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D767-0-0] [lt=7][errcode=-4076] fail to execute is_primary_tenant_ls_balance_finished_(ret=-4076, ret="OB_NEED_WAIT", tenant_id=1002) [2024-12-16 17:49:39.818040] WDIAG [RS] check_tenant_ls_balance (ob_root_utils.cpp:2059) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D767-0-0] [lt=7][errcode=-4076] fail to execute is_ls_balance_finished(ret=-4076, ret="OB_NEED_WAIT", tenant_id=1002) [2024-12-16 17:49:39.818047] WDIAG [RS] check_ls_balance_and_commit_rs_job (ob_root_utils.cpp:2123) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D767-0-0] [lt=7][errcode=-4076] fail to execute check_tenant_ls_balance(ret=-4076, ret="OB_NEED_WAIT", tenant_id=1002) [2024-12-16 17:49:39.818055] WDIAG [RS] check_shrink_resource_pool_finished_by_tenant_ (ob_shrink_expand_resource_pool_checker.cpp:148) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D767-0-0] [lt=7][errcode=-4076] fail to execute check_ls_balance_and_commit_rs_job(ret=-4076, ret="OB_NEED_WAIT", tenant_id=1002) [2024-12-16 17:49:39.818063] WDIAG [RS] check (ob_shrink_expand_resource_pool_checker.cpp:95) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D767-0-0] [lt=7][errcode=0] fail to check shrink resource pool finish(ret=0, ret="OB_SUCCESS", tmp_ret=-4076, tmp_ret="OB_NEED_WAIT", tenant_id=1002) [2024-12-16 17:49:39.818073] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:100) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D767-0-0] [lt=8] finish check shrink resource pool(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:39.818081] INFO [RS] check (ob_alter_primary_zone_checker.cpp:49) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D767-0-0] [lt=7] start check primary zone [2024-12-16 17:49:39.818095] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D768-0-0] [lt=6] start check primary zone(tenant_id=1) [2024-12-16 17:49:39.819148] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D768-0-0] [lt=7][errcode=-4008] empty result set(ret=-4008) [2024-12-16 17:49:39.819239] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=35] start check primary zone(tenant_id=1002) [2024-12-16 17:49:39.820258] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2024-12-16 17:49:39.820366] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:179) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=35] start do unit balance [2024-12-16 17:49:39.821375] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [128528][RootBalance][T1][YB42C0A89F89-00062963E283D769-0-0] [lt=11] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:12274, add_timestamp:1734400179821348}) [2024-12-16 17:49:39.822783] WDIAG [RS] do_create_user_ls (ob_common_ls_service.cpp:269) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41A-0-0] [lt=9][errcode=-4672] failed to create user ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, locality_array=[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], create_scn={val:1734359202119679000, v:0}, palf_base_info={prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}, curr_lsn:{lsn:18446744073709551615}}, create_with_palf=false, source_tenant_id=0) [2024-12-16 17:49:39.822831] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:274) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41A-0-0] [lt=37] [COMMON_LS_SERVICE] end to create ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, cost=33316) [2024-12-16 17:49:39.822844] WDIAG [RS] try_create_ls_ (ob_common_ls_service.cpp:179) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41A-0-0] [lt=12][errcode=-4672] failed to create new ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", status_info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, recovery_stat={tenant_id:1002, ls_id:{id:1003}, sync_scn:{val:1, v:0}, readable_scn:{val:1, v:0}, create_scn:{val:1734359202119679000, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:9223372036854775807, config_seq:-1}}) [2024-12-16 17:49:39.822864] WDIAG [RS] do_work (ob_common_ls_service.cpp:117) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41A-0-0] [lt=17][errcode=0] failed to create ls(ret=0, ret="OB_SUCCESS", tmp_ret=-4672, tmp_ret="OB_REPLICA_NUM_NOT_ENOUGH", user_tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:39.827864] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41A-0-0] [lt=41] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}, {unit_group_id:1002, unit_status:0, ls_group_ids:[]}]) [2024-12-16 17:49:39.830309] INFO [RS] do_work (ob_common_ls_service.cpp:138) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41A-0-0] [lt=29] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:39.838009] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128781][T1_L0_G0][T1][YB42C0A89F89-00062963E2039F07-0-0] [lt=18] [DDL] execute ddl like stmt(ret=0, cost=9, ddl_arg=NULL) [2024-12-16 17:49:39.861198] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:363) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=22] start distribute for server status change [2024-12-16 17:49:39.861290] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:414) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=28] finish distribute for server status change(ret=0) [2024-12-16 17:49:39.861303] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:213) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=13] finish do unit balance(ret=0) [2024-12-16 17:49:39.897589] INFO [RS.LB] tenant_group_balance (ob_server_balancer.cpp:128) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=7] start to do tenant group unit balance [2024-12-16 17:49:39.897785] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:930) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=35] start to do tenant group unit balance(zone=zone1) [2024-12-16 17:49:39.897817] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7935) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=17] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone1", server_disk_statistic_array_:[{server_:"192.168.159.141:2882", disk_in_use_:0, disk_total_:8589934592, wild_server_:false}, {server_:"192.168.159.137:2882", disk_in_use_:287309824, disk_total_:8589934592, wild_server_:false}], over_disk_waterlevel_:false}) [2024-12-16 17:49:39.897890] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1084) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=62] generate tenant array finish(ret=0, zone=zone1, standalone_tenant_array=[1001, 1002], group_tenant_array=[]) [2024-12-16 17:49:39.897929] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=14] generate available servers finish(ret=0, zone=zone1, available_servers=["192.168.159.137:2882", "192.168.159.141:2882"]) [2024-12-16 17:49:39.897958] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=12] generate available servers finish(ret=0, zone=zone1, available_servers=["192.168.159.137:2882", "192.168.159.141:2882"]) [2024-12-16 17:49:39.897971] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5311) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=10] start do non-tenantgroup unit balance task(zone=zone1) [2024-12-16 17:49:39.897985] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5640) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=8] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone1, cpu_weights=2.429022082076017497e-01, mem_weights=6.056782334527212042e-01, log_disk_weights=1.514195583396769074e-01, data_disk_weights=0.000000000000000000e+00, available_servers=["192.168.159.137:2882", "192.168.159.141:2882"]) [2024-12-16 17:49:39.898027] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4547) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=21] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"192.168.159.137:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:7, max_cpu:7, memory_size:"7GB", log_disk_size:"4GB", data_disk_size:0, min_iops:-2, max_iops:-2, iops_weight:6, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:7, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:7, cpu_assigned_max:7, mem_capacity:"7GB", mem_assigned:"7GB", mem_in_use:0GB, log_disk_capacity:14GB, log_disk_assigned:4GB, log_disk_in_use:1.375GB, data_disk_capacity:8GB, data_disk_assigned:0GB, data_disk_in_use:0.267578125GB}, inter_ttg_load_value_:"7.552106804876674850e-01"}, {server_:"192.168.159.141:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:4, max_cpu:4, memory_size:"5GB", log_disk_size:"2GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:3, max_net_bandwidth:"-1.86264515e-09GB", net_bandwidth_weight:4, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:4, cpu_assigned_max:4, mem_capacity:"7GB", mem_assigned:"5GB", mem_in_use:0GB, log_disk_capacity:14GB, log_disk_assigned:2GB, log_disk_in_use:0GB, data_disk_capacity:8GB, data_disk_assigned:0GB, data_disk_in_use:0GB}, inter_ttg_load_value_:"5.149842271280108408e-01"}], available_servers=["192.168.159.137:2882", "192.168.159.141:2882"], wild_server_loads=[], over_server_loads=[{server_:"192.168.159.137:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:7, max_cpu:7, memory_size:"7GB", log_disk_size:"4GB", data_disk_size:0, min_iops:-2, max_iops:-2, iops_weight:6, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:7, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:7, cpu_assigned_max:7, mem_capacity:"7GB", mem_assigned:"7GB", mem_in_use:0GB, log_disk_capacity:14GB, log_disk_assigned:4GB, log_disk_in_use:1.375GB, data_disk_capacity:8GB, data_disk_assigned:0GB, data_disk_in_use:0.267578125GB}, inter_ttg_load_value_:"7.552106804876674850e-01"}], under_server_loads=[{server_:"192.168.159.141:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:4, max_cpu:4, memory_size:"5GB", log_disk_size:"2GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:3, max_net_bandwidth:"-1.86264515e-09GB", net_bandwidth_weight:4, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:4, cpu_assigned_max:4, mem_capacity:"7GB", mem_assigned:"5GB", mem_in_use:0GB, log_disk_capacity:14GB, log_disk_assigned:2GB, log_disk_in_use:0GB, data_disk_capacity:8GB, data_disk_assigned:0GB, data_disk_in_use:0GB}, inter_ttg_load_value_:"5.149842271280108408e-01"}]) [2024-12-16 17:49:39.898115] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8126) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=87] check server over disk waterlevel(zone=zone1, disk_statistic={server_:"192.168.159.141:2882", disk_in_use_:0, disk_total_:8589934592, wild_server_:false}, over_disk_waterlevel=false) [2024-12-16 17:49:39.898131] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8132) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=14] check all available servers over disk waterlevel(zone=zone1, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"192.168.159.141:2882", disk_in_use_:0, disk_total_:8589934592, wild_server_:false}, {server_:"192.168.159.137:2882", disk_in_use_:287309824, disk_total_:8589934592, wild_server_:false}]) [2024-12-16 17:49:39.898153] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5293) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=12] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=2, not_grant_units.count()=0) [2024-12-16 17:49:39.898169] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5348) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=14] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone1, upper_lmt=6.668523264982311183e-01, wild_server_loads.count()=0, over_server_loads.count()=1, under_server_loads.count()=1, standalone_units=[{unit_id:1001, tenant_id:1002, resource_pool_id:1001, unit_config:{unit_config_id:1001, name:"unit_1", resource:{min_cpu:4, max_cpu:4, memory_size:"5GB", log_disk_size:"2GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x2b45bca19b38, unit_config:0x2b45be5f7d18, pool:0x2b4615fe5688}, {unit_id:1004, tenant_id:1002, resource_pool_id:1001, unit_config:{unit_config_id:1001, name:"unit_1", resource:{min_cpu:4, max_cpu:4, memory_size:"5GB", log_disk_size:"2GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x2b45bca198b0, unit_config:0x2b45be5f7d18, pool:0x2b4615fe5688}], not_grant_units=[], available_servers=["192.168.159.137:2882", "192.168.159.141:2882"]) [2024-12-16 17:49:39.898218] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:930) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=28] start to do tenant group unit balance(zone=zone2) [2024-12-16 17:49:39.898232] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7935) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=9] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone2", server_disk_statistic_array_:[{server_:"192.168.159.142:2882", disk_in_use_:0, disk_total_:8589934592, wild_server_:false}, {server_:"192.168.159.138:2882", disk_in_use_:251658240, disk_total_:8589934592, wild_server_:false}], over_disk_waterlevel_:false}) [2024-12-16 17:49:39.898250] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1084) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=12] generate tenant array finish(ret=0, zone=zone2, standalone_tenant_array=[1001, 1002], group_tenant_array=[]) [2024-12-16 17:49:39.898270] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=9] generate available servers finish(ret=0, zone=zone2, available_servers=["192.168.159.138:2882", "192.168.159.142:2882"]) [2024-12-16 17:49:39.898289] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=10] generate available servers finish(ret=0, zone=zone2, available_servers=["192.168.159.138:2882", "192.168.159.142:2882"]) [2024-12-16 17:49:39.898300] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5311) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=8] start do non-tenantgroup unit balance task(zone=zone2) [2024-12-16 17:49:39.898308] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5640) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=5] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone2, cpu_weights=2.429022082076017497e-01, mem_weights=6.056782334527212042e-01, log_disk_weights=1.514195583396769074e-01, data_disk_weights=0.000000000000000000e+00, available_servers=["192.168.159.138:2882", "192.168.159.142:2882"]) [2024-12-16 17:49:39.898327] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4547) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=12] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"192.168.159.138:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:7, max_cpu:7, memory_size:"7GB", log_disk_size:"4GB", data_disk_size:0, min_iops:-2, max_iops:-2, iops_weight:6, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:7, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:7, cpu_assigned_max:7, mem_capacity:"7GB", mem_assigned:"7GB", mem_in_use:0GB, log_disk_capacity:14GB, log_disk_assigned:4GB, log_disk_in_use:1.375GB, data_disk_capacity:8GB, data_disk_assigned:0GB, data_disk_in_use:0.234375GB}, inter_ttg_load_value_:"7.552106804876674850e-01"}, {server_:"192.168.159.142:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:4, max_cpu:4, memory_size:"5GB", log_disk_size:"2GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:3, max_net_bandwidth:"-1.86264515e-09GB", net_bandwidth_weight:4, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:4, cpu_assigned_max:4, mem_capacity:"7GB", mem_assigned:"5GB", mem_in_use:0GB, log_disk_capacity:14GB, log_disk_assigned:2GB, log_disk_in_use:0GB, data_disk_capacity:8GB, data_disk_assigned:0GB, data_disk_in_use:0GB}, inter_ttg_load_value_:"5.149842271280108408e-01"}], available_servers=["192.168.159.138:2882", "192.168.159.142:2882"], wild_server_loads=[], over_server_loads=[{server_:"192.168.159.138:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:7, max_cpu:7, memory_size:"7GB", log_disk_size:"4GB", data_disk_size:0, min_iops:-2, max_iops:-2, iops_weight:6, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:7, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:7, cpu_assigned_max:7, mem_capacity:"7GB", mem_assigned:"7GB", mem_in_use:0GB, log_disk_capacity:14GB, log_disk_assigned:4GB, log_disk_in_use:1.375GB, data_disk_capacity:8GB, data_disk_assigned:0GB, data_disk_in_use:0.234375GB}, inter_ttg_load_value_:"7.552106804876674850e-01"}], under_server_loads=[{server_:"192.168.159.142:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:4, max_cpu:4, memory_size:"5GB", log_disk_size:"2GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:3, max_net_bandwidth:"-1.86264515e-09GB", net_bandwidth_weight:4, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:4, cpu_assigned_max:4, mem_capacity:"7GB", mem_assigned:"5GB", mem_in_use:0GB, log_disk_capacity:14GB, log_disk_assigned:2GB, log_disk_in_use:0GB, data_disk_capacity:8GB, data_disk_assigned:0GB, data_disk_in_use:0GB}, inter_ttg_load_value_:"5.149842271280108408e-01"}]) [2024-12-16 17:49:39.898401] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8126) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=73] check server over disk waterlevel(zone=zone2, disk_statistic={server_:"192.168.159.142:2882", disk_in_use_:0, disk_total_:8589934592, wild_server_:false}, over_disk_waterlevel=false) [2024-12-16 17:49:39.898414] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8132) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=13] check all available servers over disk waterlevel(zone=zone2, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"192.168.159.142:2882", disk_in_use_:0, disk_total_:8589934592, wild_server_:false}, {server_:"192.168.159.138:2882", disk_in_use_:251658240, disk_total_:8589934592, wild_server_:false}]) [2024-12-16 17:49:39.898431] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5293) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=11] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=2, not_grant_units.count()=0) [2024-12-16 17:49:39.898486] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5348) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=54] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone2, upper_lmt=6.668523264982311183e-01, wild_server_loads.count()=0, over_server_loads.count()=1, under_server_loads.count()=1, standalone_units=[{unit_id:1002, tenant_id:1002, resource_pool_id:1001, unit_config:{unit_config_id:1001, name:"unit_1", resource:{min_cpu:4, max_cpu:4, memory_size:"5GB", log_disk_size:"2GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x2b45bca19a60, unit_config:0x2b45be5f7d18, pool:0x2b4615fe5688}, {unit_id:1005, tenant_id:1002, resource_pool_id:1001, unit_config:{unit_config_id:1001, name:"unit_1", resource:{min_cpu:4, max_cpu:4, memory_size:"5GB", log_disk_size:"2GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x2b45bca197d8, unit_config:0x2b45be5f7d18, pool:0x2b4615fe5688}], not_grant_units=[], available_servers=["192.168.159.138:2882", "192.168.159.142:2882"]) [2024-12-16 17:49:39.898536] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:930) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=29] start to do tenant group unit balance(zone=zone3) [2024-12-16 17:49:39.898549] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7935) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=9] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone3", server_disk_statistic_array_:[{server_:"192.168.159.143:2882", disk_in_use_:0, disk_total_:8589934592, wild_server_:false}, {server_:"192.168.159.139:2882", disk_in_use_:266338304, disk_total_:8589934592, wild_server_:false}], over_disk_waterlevel_:false}) [2024-12-16 17:49:39.898567] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1084) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=13] generate tenant array finish(ret=0, zone=zone3, standalone_tenant_array=[1001, 1002], group_tenant_array=[]) [2024-12-16 17:49:39.898585] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=7] generate available servers finish(ret=0, zone=zone3, available_servers=["192.168.159.139:2882", "192.168.159.143:2882"]) [2024-12-16 17:49:39.898603] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=8] generate available servers finish(ret=0, zone=zone3, available_servers=["192.168.159.139:2882", "192.168.159.143:2882"]) [2024-12-16 17:49:39.898692] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5311) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=84] start do non-tenantgroup unit balance task(zone=zone3) [2024-12-16 17:49:39.898724] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5640) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=12] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone3, cpu_weights=2.429022082076017497e-01, mem_weights=6.056782334527212042e-01, log_disk_weights=1.514195583396769074e-01, data_disk_weights=0.000000000000000000e+00, available_servers=["192.168.159.139:2882", "192.168.159.143:2882"]) [2024-12-16 17:49:39.898750] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4547) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=18] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"192.168.159.139:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:7, max_cpu:7, memory_size:"7GB", log_disk_size:"4GB", data_disk_size:0, min_iops:-2, max_iops:-2, iops_weight:6, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:7, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:7, cpu_assigned_max:7, mem_capacity:"7GB", mem_assigned:"7GB", mem_in_use:0GB, log_disk_capacity:14GB, log_disk_assigned:4GB, log_disk_in_use:1.375GB, data_disk_capacity:8GB, data_disk_assigned:0GB, data_disk_in_use:0.248046875GB}, inter_ttg_load_value_:"7.552106804876674850e-01"}, {server_:"192.168.159.143:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:4, max_cpu:4, memory_size:"5GB", log_disk_size:"2GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:3, max_net_bandwidth:"-1.86264515e-09GB", net_bandwidth_weight:4, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:4, cpu_assigned_max:4, mem_capacity:"7GB", mem_assigned:"5GB", mem_in_use:0GB, log_disk_capacity:14GB, log_disk_assigned:2GB, log_disk_in_use:0GB, data_disk_capacity:8GB, data_disk_assigned:0GB, data_disk_in_use:0GB}, inter_ttg_load_value_:"5.149842271280108408e-01"}], available_servers=["192.168.159.139:2882", "192.168.159.143:2882"], wild_server_loads=[], over_server_loads=[{server_:"192.168.159.139:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:7, max_cpu:7, memory_size:"7GB", log_disk_size:"4GB", data_disk_size:0, min_iops:-2, max_iops:-2, iops_weight:6, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:7, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:7, cpu_assigned_max:7, mem_capacity:"7GB", mem_assigned:"7GB", mem_in_use:0GB, log_disk_capacity:14GB, log_disk_assigned:4GB, log_disk_in_use:1.375GB, data_disk_capacity:8GB, data_disk_assigned:0GB, data_disk_in_use:0.248046875GB}, inter_ttg_load_value_:"7.552106804876674850e-01"}], under_server_loads=[{server_:"192.168.159.143:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:4, max_cpu:4, memory_size:"5GB", log_disk_size:"2GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:3, max_net_bandwidth:"-1.86264515e-09GB", net_bandwidth_weight:4, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:4, cpu_assigned_max:4, mem_capacity:"7GB", mem_assigned:"5GB", mem_in_use:0GB, log_disk_capacity:14GB, log_disk_assigned:2GB, log_disk_in_use:0GB, data_disk_capacity:8GB, data_disk_assigned:0GB, data_disk_in_use:0GB}, inter_ttg_load_value_:"5.149842271280108408e-01"}]) [2024-12-16 17:49:39.898830] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8126) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=78] check server over disk waterlevel(zone=zone3, disk_statistic={server_:"192.168.159.143:2882", disk_in_use_:0, disk_total_:8589934592, wild_server_:false}, over_disk_waterlevel=false) [2024-12-16 17:49:39.898844] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8132) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=13] check all available servers over disk waterlevel(zone=zone3, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"192.168.159.143:2882", disk_in_use_:0, disk_total_:8589934592, wild_server_:false}, {server_:"192.168.159.139:2882", disk_in_use_:266338304, disk_total_:8589934592, wild_server_:false}]) [2024-12-16 17:49:39.898862] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5293) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=11] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=2, not_grant_units.count()=0) [2024-12-16 17:49:39.898874] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5348) [128528][RootBalance][T0][YB42C0A89F89-00062963E283D769-0-0] [lt=11] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone3, upper_lmt=6.668523264982311183e-01, wild_server_loads.count()=0, over_server_loads.count()=1, under_server_loads.count()=1, standalone_units=[{unit_id:1003, tenant_id:1002, resource_pool_id:1001, unit_config:{unit_config_id:1001, name:"unit_1", resource:{min_cpu:4, max_cpu:4, memory_size:"5GB", log_disk_size:"2GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x2b45bca19988, unit_config:0x2b45be5f7d18, pool:0x2b4615fe5688}, {unit_id:1006, tenant_id:1002, resource_pool_id:1001, unit_config:{unit_config_id:1001, name:"unit_1", resource:{min_cpu:4, max_cpu:4, memory_size:"5GB", log_disk_size:"2GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x2b45bca19700, unit_config:0x2b45be5f7d18, pool:0x2b4615fe5688}], not_grant_units=[], available_servers=["192.168.159.139:2882", "192.168.159.143:2882"]) [2024-12-16 17:49:39.914090] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8B-00062963DFDFA021-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=21, ddl_arg=NULL) [2024-12-16 17:49:40.028505] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129809][T1002_StandbySc][T1002][YB42C0A89F89-00062963E533F95A-0-0] [lt=25] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:40.040110] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8E-0006296420107356-0-0] [lt=66] [DDL] execute ddl like stmt(ret=0, cost=21, ddl_arg=NULL) [2024-12-16 17:49:40.160986] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [128655][T1_StandbySchem][T1][YB42C0A89F89-00062963DD93FAD2-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:40.253263] INFO [RS] check_lost_replicas (ob_lost_replica_checker.cpp:117) [128527][LostRepCheck][T0][Y0-0000000000000000-0-0] [lt=24] start checking lost replicas [2024-12-16 17:49:40.277796] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F805-0-0] [lt=44] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202081794000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202110029000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:0, operation_type_:0, create_scn:{val:1734359202119679000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:40.280699] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F805-0-0] [lt=156] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-16 17:49:40.280735] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F805-0-0] [lt=35] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:40.417126] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128852][T1_L0_G0][T1][YB42C0A89F8D-000629641D38A497-0-0] [lt=17] [DDL] execute ddl like stmt(ret=0, cost=54, ddl_arg=NULL) [2024-12-16 17:49:40.513743] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F42A-0-0] [lt=18] is primary tenant, no need check readable_scn [2024-12-16 17:49:40.526793] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F42A-0-0] [lt=32] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.159.137:2882", readable_scn:{val:1734400180469768000, v:0}}, {server:"192.168.159.138:2882", readable_scn:{val:1734400180469768000, v:0}}, {server:"192.168.159.139:2882", readable_scn:{val:1734400179970082005, v:0}}], last_dump_ts_=1734400175328908, force_dump=false) [2024-12-16 17:49:40.526834] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F42A-0-0] [lt=35] is primary tenant, no need check readable_scn [2024-12-16 17:49:40.540224] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F42A-0-0] [lt=9] is primary tenant, no need check readable_scn [2024-12-16 17:49:40.586593] INFO [RS] do_work (ob_common_ls_service.cpp:138) [128656][T1_COMMONLSSe][T1][YB42C0A89F89-00062963E043FA83-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:40.740827] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1734400179469249000, replayable_scn:1734400179469249000, readable_scn:1734359204309426002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1734400180737609, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1734400179469249000, replayable_scn:1734400179469249000, readable_scn:1734359204309426002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1734400180737609, ora_rowscn_=1734400180517286000, content_changed=true) [2024-12-16 17:49:40.744157] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=71] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.159.137:2882") [2024-12-16 17:49:40.744183] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=25] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="192.168.159.138:2882") [2024-12-16 17:49:40.744192] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="192.168.159.139:2882") [2024-12-16 17:49:40.744199] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=3, dest="192.168.159.141:2882") [2024-12-16 17:49:40.744206] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=4, dest="192.168.159.142:2882") [2024-12-16 17:49:40.744213] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=5, dest="192.168.159.143:2882") [2024-12-16 17:49:40.778200] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129974][T1001_StandbySc][T1001][YB42C0A89F89-00062963E5D3F959-0-0] [lt=29] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:40.801695] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [129880][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=22] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1734400180737609, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1734400179469249000, replayable_scn:1734400179469249000, readable_scn:1734359204309426002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-16 17:49:40.833751] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41B-0-0] [lt=19] [COMMON_LS_SERVICE] start to create ls(info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, create_scn={val:1734359202119679000, v:0}) [2024-12-16 17:49:40.845048] WDIAG [RS] do_create_user_ls (ob_common_ls_service.cpp:269) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41B-0-0] [lt=27][errcode=-4672] failed to create user ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, locality_array=[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], create_scn={val:1734359202119679000, v:0}, palf_base_info={prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}, curr_lsn:{lsn:18446744073709551615}}, create_with_palf=false, source_tenant_id=0) [2024-12-16 17:49:40.845142] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:274) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41B-0-0] [lt=85] [COMMON_LS_SERVICE] end to create ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, cost=11343) [2024-12-16 17:49:40.845156] WDIAG [RS] try_create_ls_ (ob_common_ls_service.cpp:179) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41B-0-0] [lt=13][errcode=-4672] failed to create new ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", status_info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, recovery_stat={tenant_id:1002, ls_id:{id:1003}, sync_scn:{val:1, v:0}, readable_scn:{val:1, v:0}, create_scn:{val:1734359202119679000, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:9223372036854775807, config_seq:-1}}) [2024-12-16 17:49:40.845177] WDIAG [RS] do_work (ob_common_ls_service.cpp:117) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41B-0-0] [lt=18][errcode=0] failed to create ls(ret=0, ret="OB_SUCCESS", tmp_ret=-4672, tmp_ret="OB_REPLICA_NUM_NOT_ENOUGH", user_tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:40.849705] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41B-0-0] [lt=43] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}, {unit_group_id:1002, unit_status:0, ls_group_ids:[]}]) [2024-12-16 17:49:40.852331] INFO [RS] do_work (ob_common_ls_service.cpp:138) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41B-0-0] [lt=29] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:41.029959] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129809][T1002_StandbySc][T1002][YB42C0A89F89-00062963E533F95B-0-0] [lt=44] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:41.059842] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F89-00062963E103A9E8-0-0] [lt=106] [DDL] execute ddl like stmt(ret=0, cost=21, ddl_arg=NULL) [2024-12-16 17:49:41.066731] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [128685][T1_HBService][T1][YB42C0A89F89-00062963E023AA12-0-0] [lt=43] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:41.130376] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [130096][T1001_FrzInfoDe][T1001][YB42C0A89F89-00062963E7F36926-0-0] [lt=29] get number(ret=0, service_type_=0, range=1, base_id=1734400181130111000, start_id=1734400181130111000, end_id=1734400181130111001) [2024-12-16 17:49:41.135787] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [130096][T1001_FrzInfoDe][T1001][YB42C0A89F89-00062963E7F36926-0-0] [lt=31] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1734400181130111000, v:0}, tenant_id=1001) [2024-12-16 17:49:41.155323] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8F-000629642BDC7AD7-0-0] [lt=33] [DDL] execute ddl like stmt(ret=0, cost=19, ddl_arg=NULL) [2024-12-16 17:49:41.161136] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [128655][T1_StandbySchem][T1][YB42C0A89F89-00062963DD93FAD3-0-0] [lt=33] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:41.284258] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F806-0-0] [lt=42] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202081794000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202110029000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:0, operation_type_:0, create_scn:{val:1734359202119679000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:41.287601] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F806-0-0] [lt=102] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-16 17:49:41.287686] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F806-0-0] [lt=83] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:41.321613] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8A-00062963DFD99B2C-0-0] [lt=35] [DDL] execute ddl like stmt(ret=0, cost=84, ddl_arg=NULL) [2024-12-16 17:49:41.381223] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8D-000629641B98C90D-0-0] [lt=116] [DDL] execute ddl like stmt(ret=0, cost=83, ddl_arg=NULL) [2024-12-16 17:49:41.482694] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128853][T1_L0_G0][T1][YB42C0A89F8B-00062963E0AF7AA0-0-0] [lt=38] [DDL] execute ddl like stmt(ret=0, cost=12, ddl_arg=NULL) [2024-12-16 17:49:41.551895] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128852][T1_L0_G0][T1][YB42C0A89F8E-0006296421A04F44-0-0] [lt=42] [DDL] execute ddl like stmt(ret=0, cost=19, ddl_arg=NULL) [2024-12-16 17:49:41.559873] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F42B-0-0] [lt=34] is primary tenant, no need check readable_scn [2024-12-16 17:49:41.586525] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F42B-0-0] [lt=20] is primary tenant, no need check readable_scn [2024-12-16 17:49:41.588221] INFO [RS] do_work (ob_common_ls_service.cpp:138) [128656][T1_COMMONLSSe][T1][YB42C0A89F89-00062963E043FA84-0-0] [lt=103] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:41.602139] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F42B-0-0] [lt=24] is primary tenant, no need check readable_scn [2024-12-16 17:49:41.744436] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.159.137:2882") [2024-12-16 17:49:41.744476] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=38] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="192.168.159.138:2882") [2024-12-16 17:49:41.744488] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="192.168.159.139:2882") [2024-12-16 17:49:41.744495] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=3, dest="192.168.159.141:2882") [2024-12-16 17:49:41.744502] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=4, dest="192.168.159.142:2882") [2024-12-16 17:49:41.744509] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=5, dest="192.168.159.143:2882") [2024-12-16 17:49:41.751025] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [128684][T1_HBService][T1][YB42C0A89F89-00062963E003A9C6-0-0] [lt=23] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:41.781130] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129974][T1001_StandbySc][T1001][YB42C0A89F89-00062963E5D3F95A-0-0] [lt=40] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:41.855734] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41C-0-0] [lt=23] [COMMON_LS_SERVICE] start to create ls(info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, create_scn={val:1734359202119679000, v:0}) [2024-12-16 17:49:41.875137] WDIAG [RS] do_create_user_ls (ob_common_ls_service.cpp:269) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41C-0-0] [lt=21][errcode=-4672] failed to create user ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, locality_array=[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], create_scn={val:1734359202119679000, v:0}, palf_base_info={prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}, curr_lsn:{lsn:18446744073709551615}}, create_with_palf=false, source_tenant_id=0) [2024-12-16 17:49:41.875209] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:274) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41C-0-0] [lt=59] [COMMON_LS_SERVICE] end to create ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, cost=19431) [2024-12-16 17:49:41.875224] WDIAG [RS] try_create_ls_ (ob_common_ls_service.cpp:179) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41C-0-0] [lt=14][errcode=-4672] failed to create new ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", status_info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, recovery_stat={tenant_id:1002, ls_id:{id:1003}, sync_scn:{val:1, v:0}, readable_scn:{val:1, v:0}, create_scn:{val:1734359202119679000, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:9223372036854775807, config_seq:-1}}) [2024-12-16 17:49:41.875244] WDIAG [RS] do_work (ob_common_ls_service.cpp:117) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41C-0-0] [lt=17][errcode=0] failed to create ls(ret=0, ret="OB_SUCCESS", tmp_ret=-4672, tmp_ret="OB_REPLICA_NUM_NOT_ENOUGH", user_tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:41.880530] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41C-0-0] [lt=44] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}, {unit_group_id:1002, unit_status:0, ls_group_ids:[]}]) [2024-12-16 17:49:41.882068] INFO [RS] try_update_primary_ip_list (ob_common_ls_service.cpp:361) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41C-0-0] [lt=18] log restore source is empty, just skip(ret=-4018, user_tenant_id=1002) [2024-12-16 17:49:41.886835] INFO [RS] do_work (ob_common_ls_service.cpp:138) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41C-0-0] [lt=9] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:41.917039] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8B-00062963DFDFA022-0-0] [lt=28] [DDL] execute ddl like stmt(ret=0, cost=20, ddl_arg=NULL) [2024-12-16 17:49:42.030141] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129809][T1002_StandbySc][T1002][YB42C0A89F89-00062963E533F95C-0-0] [lt=26] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:42.042600] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8E-0006296420107357-0-0] [lt=50] [DDL] execute ddl like stmt(ret=0, cost=72, ddl_arg=NULL) [2024-12-16 17:49:42.161467] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [128655][T1_StandbySchem][T1][YB42C0A89F89-00062963DD93FAD4-0-0] [lt=40] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:42.291580] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F807-0-0] [lt=48] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202081794000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202110029000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:0, operation_type_:0, create_scn:{val:1734359202119679000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:42.295567] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F807-0-0] [lt=161] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-16 17:49:42.295586] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F807-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:42.590037] INFO [RS] do_work (ob_common_ls_service.cpp:138) [128656][T1_COMMONLSSe][T1][YB42C0A89F89-00062963E043FA85-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:42.616880] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F42C-0-0] [lt=20] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.159.137:2882", readable_scn:{val:1734400182473193000, v:0}}, {server:"192.168.159.138:2882", readable_scn:{val:1734400182473193001, v:0}}, {server:"192.168.159.139:2882", readable_scn:{val:1734400182473193001, v:0}}], last_dump_ts_=1734400177387597, force_dump=false) [2024-12-16 17:49:42.616916] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F42C-0-0] [lt=29] is primary tenant, no need check readable_scn [2024-12-16 17:49:42.629718] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F42C-0-0] [lt=8] is primary tenant, no need check readable_scn [2024-12-16 17:49:42.638884] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F42C-0-0] [lt=91] is primary tenant, no need check readable_scn [2024-12-16 17:49:42.744447] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.159.137:2882") [2024-12-16 17:49:42.744473] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=26] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="192.168.159.138:2882") [2024-12-16 17:49:42.744482] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="192.168.159.139:2882") [2024-12-16 17:49:42.744489] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=3, dest="192.168.159.141:2882") [2024-12-16 17:49:42.744496] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=4, dest="192.168.159.142:2882") [2024-12-16 17:49:42.744503] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=5, dest="192.168.159.143:2882") [2024-12-16 17:49:42.746217] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-16 17:49:42.782844] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129974][T1001_StandbySc][T1001][YB42C0A89F89-00062963E5D3F95B-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:42.892255] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41D-0-0] [lt=40] [COMMON_LS_SERVICE] start to create ls(info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, create_scn={val:1734359202119679000, v:0}) [2024-12-16 17:49:42.917888] WDIAG [RS] do_create_user_ls (ob_common_ls_service.cpp:269) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41D-0-0] [lt=41][errcode=-4672] failed to create user ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, locality_array=[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], create_scn={val:1734359202119679000, v:0}, palf_base_info={prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}, curr_lsn:{lsn:18446744073709551615}}, create_with_palf=false, source_tenant_id=0) [2024-12-16 17:49:42.918027] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:274) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41D-0-0] [lt=124] [COMMON_LS_SERVICE] end to create ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, cost=25714) [2024-12-16 17:49:42.918043] WDIAG [RS] try_create_ls_ (ob_common_ls_service.cpp:179) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41D-0-0] [lt=15][errcode=-4672] failed to create new ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", status_info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, recovery_stat={tenant_id:1002, ls_id:{id:1003}, sync_scn:{val:1, v:0}, readable_scn:{val:1, v:0}, create_scn:{val:1734359202119679000, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:9223372036854775807, config_seq:-1}}) [2024-12-16 17:49:42.918064] WDIAG [RS] do_work (ob_common_ls_service.cpp:117) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41D-0-0] [lt=18][errcode=0] failed to create ls(ret=0, ret="OB_SUCCESS", tmp_ret=-4672, tmp_ret="OB_REPLICA_NUM_NOT_ENOUGH", user_tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:42.925731] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41D-0-0] [lt=50] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}, {unit_group_id:1002, unit_status:0, ls_group_ids:[]}]) [2024-12-16 17:49:42.935703] INFO [RS] do_work (ob_common_ls_service.cpp:138) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41D-0-0] [lt=56] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:43.030311] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129809][T1002_StandbySc][T1002][YB42C0A89F89-00062963E533F95D-0-0] [lt=25] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:43.060840] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F89-00062963E103A9E9-0-0] [lt=81] [DDL] execute ddl like stmt(ret=0, cost=20, ddl_arg=NULL) [2024-12-16 17:49:43.073223] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [128685][T1_HBService][T1][YB42C0A89F89-00062963E023AA13-0-0] [lt=43] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:43.157780] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8F-000629642BDC7AD8-0-0] [lt=29] [DDL] execute ddl like stmt(ret=0, cost=20, ddl_arg=NULL) [2024-12-16 17:49:43.161637] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [128655][T1_StandbySchem][T1][YB42C0A89F89-00062963DD93FAD5-0-0] [lt=50] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:43.236889] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128859][T1_L0_G0][T1][YB42C0A89F8F-000629642D9C571C-0-0] [lt=17] [DDL] execute ddl like stmt(ret=0, cost=13, ddl_arg=NULL) [2024-12-16 17:49:43.299273] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F808-0-0] [lt=151] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202081794000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202110029000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:0, operation_type_:0, create_scn:{val:1734359202119679000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:43.302290] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F808-0-0] [lt=171] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-16 17:49:43.302329] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F808-0-0] [lt=37] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:43.322982] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8A-00062963DFD99B2D-0-0] [lt=46] [DDL] execute ddl like stmt(ret=0, cost=23, ddl_arg=NULL) [2024-12-16 17:49:43.384705] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8D-000629641B98C90E-0-0] [lt=39] [DDL] execute ddl like stmt(ret=0, cost=189, ddl_arg=NULL) [2024-12-16 17:49:43.425793] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [128532][DBMS_JOB_MASTER][T0][YB42C0A89F89-00062963DE235926-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2024-12-16 17:49:43.427556] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [128532][DBMS_JOB_MASTER][T0][YB42C0A89F89-00062963DE235926-0-0] [lt=74] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1734400183424347, delay:5000000, job_id:0, tenant_id:1002}, job_key=0x2b45b422a0e0, job_infos=[]) [2024-12-16 17:49:43.427585] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [128532][DBMS_JOB_MASTER][T0][YB42C0A89F89-00062963DE235926-0-0] [lt=26] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1734400188427582, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-16 17:49:43.535445] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [128532][DBMS_JOB_MASTER][T0][YB42C0A89F89-00062963DE235926-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2024-12-16 17:49:43.537898] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [128532][DBMS_JOB_MASTER][T0][YB42C0A89F89-00062963DE235926-0-0] [lt=23] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1734400183534510, delay:5000000, job_id:0, tenant_id:1001}, job_key=0x2b45b422a0a0, job_infos=[]) [2024-12-16 17:49:43.537977] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [128532][DBMS_JOB_MASTER][T0][YB42C0A89F89-00062963DE235926-0-0] [lt=75] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1734400188537973, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-16 17:49:43.537995] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [128532][DBMS_JOB_MASTER][T0][YB42C0A89F89-00062963DE235926-0-0] [lt=13] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1734400188537991, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-16 17:49:43.592599] INFO [RS] do_work (ob_common_ls_service.cpp:138) [128656][T1_COMMONLSSe][T1][YB42C0A89F89-00062963E043FA86-0-0] [lt=86] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:43.648441] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F42D-0-0] [lt=22] is primary tenant, no need check readable_scn [2024-12-16 17:49:43.666131] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F42D-0-0] [lt=26] is primary tenant, no need check readable_scn [2024-12-16 17:49:43.676006] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F42D-0-0] [lt=23] is primary tenant, no need check readable_scn [2024-12-16 17:49:43.676069] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:464) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F42D-0-0] [lt=19] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1734400181471221000, v:0}) [2024-12-16 17:49:43.742240] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1734400182473193001, replayable_scn:1734400182473193001, readable_scn:1734359204309426002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1734400183737183, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1734400182473193001, replayable_scn:1734400182473193001, readable_scn:1734359204309426002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1734400183737183, ora_rowscn_=1734400183647291001, content_changed=true) [2024-12-16 17:49:43.747530] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=40] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.159.137:2882") [2024-12-16 17:49:43.747569] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=39] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="192.168.159.138:2882") [2024-12-16 17:49:43.747578] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="192.168.159.139:2882") [2024-12-16 17:49:43.747585] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=3, dest="192.168.159.141:2882") [2024-12-16 17:49:43.747592] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=4, dest="192.168.159.142:2882") [2024-12-16 17:49:43.747599] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=5, dest="192.168.159.143:2882") [2024-12-16 17:49:43.747614] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=5331, return_code_array=[0, 0, 0, 0, 0, 0], broadcast_times=2) [2024-12-16 17:49:43.756026] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [128684][T1_HBService][T1][YB42C0A89F89-00062963E003A9C7-0-0] [lt=22] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:43.782939] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129974][T1001_StandbySc][T1001][YB42C0A89F89-00062963E5D3F95C-0-0] [lt=32] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:43.836132] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [128834][T1_FrzInfoDet][T1][YB42C0A89F89-00062963DF93694B-0-0] [lt=45] get number(ret=0, service_type_=0, range=1, base_id=1734400183831976000, start_id=1734400183831976000, end_id=1734400183831976001) [2024-12-16 17:49:43.838651] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [129880][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=28] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1734400183737183, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1734400182473193001, replayable_scn:1734400182473193001, readable_scn:1734359204309426002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-16 17:49:43.841011] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [128834][T1_FrzInfoDet][T1][YB42C0A89F89-00062963DF93694B-0-0] [lt=55] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1734400183831976000, v:0}, tenant_id=1) [2024-12-16 17:49:43.921069] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8B-00062963DFDFA023-0-0] [lt=153] [DDL] execute ddl like stmt(ret=0, cost=23, ddl_arg=NULL) [2024-12-16 17:49:43.939484] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41E-0-0] [lt=69] [COMMON_LS_SERVICE] start to create ls(info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, create_scn={val:1734359202119679000, v:0}) [2024-12-16 17:49:43.955842] WDIAG [RS] do_create_user_ls (ob_common_ls_service.cpp:269) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41E-0-0] [lt=8][errcode=-4672] failed to create user ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, locality_array=[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], create_scn={val:1734359202119679000, v:0}, palf_base_info={prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}, curr_lsn:{lsn:18446744073709551615}}, create_with_palf=false, source_tenant_id=0) [2024-12-16 17:49:43.955890] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:274) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41E-0-0] [lt=37] [COMMON_LS_SERVICE] end to create ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, cost=16349) [2024-12-16 17:49:43.955901] WDIAG [RS] try_create_ls_ (ob_common_ls_service.cpp:179) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41E-0-0] [lt=10][errcode=-4672] failed to create new ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", status_info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, recovery_stat={tenant_id:1002, ls_id:{id:1003}, sync_scn:{val:1, v:0}, readable_scn:{val:1, v:0}, create_scn:{val:1734359202119679000, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:9223372036854775807, config_seq:-1}}) [2024-12-16 17:49:43.955921] WDIAG [RS] do_work (ob_common_ls_service.cpp:117) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41E-0-0] [lt=17][errcode=0] failed to create ls(ret=0, ret="OB_SUCCESS", tmp_ret=-4672, tmp_ret="OB_REPLICA_NUM_NOT_ENOUGH", user_tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:43.961885] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41E-0-0] [lt=100] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}, {unit_group_id:1002, unit_status:0, ls_group_ids:[]}]) [2024-12-16 17:49:43.964821] INFO [RS] do_work (ob_common_ls_service.cpp:138) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41E-0-0] [lt=26] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:44.032584] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129809][T1002_StandbySc][T1002][YB42C0A89F89-00062963E533F95E-0-0] [lt=26] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:44.046014] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8E-0006296420107358-0-0] [lt=49] [DDL] execute ddl like stmt(ret=0, cost=19, ddl_arg=NULL) [2024-12-16 17:49:44.161949] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [128655][T1_StandbySchem][T1][YB42C0A89F89-00062963DD93FAD6-0-0] [lt=21] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:44.307254] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F809-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202081794000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202110029000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:0, operation_type_:0, create_scn:{val:1734359202119679000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:44.311034] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F809-0-0] [lt=128] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-16 17:49:44.311080] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F809-0-0] [lt=45] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:44.325042] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128860][T1_L0_G0][T1][YB42C0A89F8A-00062963E0A97583-0-0] [lt=138] [DDL] execute ddl like stmt(ret=0, cost=13, ddl_arg=NULL) [2024-12-16 17:49:44.594848] INFO [RS] do_work (ob_common_ls_service.cpp:138) [128656][T1_COMMONLSSe][T1][YB42C0A89F89-00062963E043FA87-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:44.681820] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F42E-0-0] [lt=14] is primary tenant, no need check readable_scn [2024-12-16 17:49:44.693446] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F42E-0-0] [lt=23] is primary tenant, no need check readable_scn [2024-12-16 17:49:44.703156] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F42E-0-0] [lt=21] is primary tenant, no need check readable_scn [2024-12-16 17:49:44.749582] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.159.137:2882") [2024-12-16 17:49:44.749608] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=26] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="192.168.159.138:2882") [2024-12-16 17:49:44.749617] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="192.168.159.139:2882") [2024-12-16 17:49:44.749624] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=3, dest="192.168.159.141:2882") [2024-12-16 17:49:44.749678] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=53] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=4, dest="192.168.159.142:2882") [2024-12-16 17:49:44.749685] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=5, dest="192.168.159.143:2882") [2024-12-16 17:49:44.783125] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129974][T1001_StandbySc][T1001][YB42C0A89F89-00062963E5D3F95D-0-0] [lt=27] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:44.974312] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41F-0-0] [lt=48] [COMMON_LS_SERVICE] start to create ls(info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, create_scn={val:1734359202119679000, v:0}) [2024-12-16 17:49:44.989056] WDIAG [RS] do_create_user_ls (ob_common_ls_service.cpp:269) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41F-0-0] [lt=8][errcode=-4672] failed to create user ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, locality_array=[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], create_scn={val:1734359202119679000, v:0}, palf_base_info={prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}, curr_lsn:{lsn:18446744073709551615}}, create_with_palf=false, source_tenant_id=0) [2024-12-16 17:49:44.989122] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:274) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41F-0-0] [lt=56] [COMMON_LS_SERVICE] end to create ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, cost=14777) [2024-12-16 17:49:44.989136] WDIAG [RS] try_create_ls_ (ob_common_ls_service.cpp:179) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41F-0-0] [lt=14][errcode=-4672] failed to create new ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", status_info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, recovery_stat={tenant_id:1002, ls_id:{id:1003}, sync_scn:{val:1, v:0}, readable_scn:{val:1, v:0}, create_scn:{val:1734359202119679000, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:9223372036854775807, config_seq:-1}}) [2024-12-16 17:49:44.989157] WDIAG [RS] do_work (ob_common_ls_service.cpp:117) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41F-0-0] [lt=17][errcode=0] failed to create ls(ret=0, ret="OB_SUCCESS", tmp_ret=-4672, tmp_ret="OB_REPLICA_NUM_NOT_ENOUGH", user_tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:44.994930] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41F-0-0] [lt=89] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}, {unit_group_id:1002, unit_status:0, ls_group_ids:[]}]) [2024-12-16 17:49:44.997480] INFO [RS] do_work (ob_common_ls_service.cpp:138) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F41F-0-0] [lt=35] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:45.032861] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129809][T1002_StandbySc][T1002][YB42C0A89F89-00062963E533F95F-0-0] [lt=86] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:45.057015] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:131) [129980][T1001_REST_SER][T1001][YB42C0A89F89-00062963E80363CD-0-0] [lt=16] [RESTORE] restore scheduler quit [2024-12-16 17:49:45.058569] INFO [SHARE] get_all_recover_table_job (ob_recover_table_persist_helper.cpp:199) [129980][T1001_REST_SER][T1001][YB42C0A89F89-00062963E80363CD-0-0] [lt=72] get targets(ret=0, jobs=[], exec_tenant_id=1001, sql=select * from __all_recover_table_job) [2024-12-16 17:49:45.060233] INFO [SHARE] get_all_import_table_jobs (ob_import_table_persist_helper.cpp:109) [129980][T1001_REST_SER][T1001][YB42C0A89F89-00062963E80363CE-0-0] [lt=23] get targets(ret=0, jobs=[], exec_tenant_id=1001, sql=select * from __all_import_table_job) [2024-12-16 17:49:45.060265] WDIAG [SHARE.SCHEMA] revert (ob_schema_mgr_cache.cpp:122) [129980][T1001_REST_SER][T1001][YB42C0A89F89-00062963E80363CF-0-0] [lt=21][errcode=0] long time to hold one guard(schema_mgr=0x2b466bc04050, tenant_id=1, version=1734362366739016, cur_timestamp=1734400185060257, ref_timestamp=1734400125053880, lbt()="0x7bcec65 0x1a6c403f 0x78e2779 0x78e28f0 0x76151c5 0xd191344 0xbfe6d2f 0x1f623704 0x1f629f7c 0x1f629758 0x1f6278be 0x2b456c0fbea5 0x2b456c40e96d") [2024-12-16 17:49:45.063972] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F89-00062963E103A9EA-0-0] [lt=45] [DDL] execute ddl like stmt(ret=0, cost=18, ddl_arg=NULL) [2024-12-16 17:49:45.064842] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:100) [129980][T1001_REST_SER][T1001][YB42C0A89F89-00062963E80363D0-0-0] [lt=15] [RESTORE] restore scheduler start [2024-12-16 17:49:45.064877] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:107) [129980][T1001_REST_SER][T1001][YB42C0A89F89-00062963E80363D1-0-0] [lt=34] [RESTORE] try process restore job [2024-12-16 17:49:45.066293] INFO [COMMON] get_jobs (ob_physical_restore_table_operator.cpp:419) [129980][T1001_REST_SER][T1001][YB42C0A89F89-00062963E80363D1-0-0] [lt=9] [RESTORE] get restore jobs(ret=0, job_cnt=0) [2024-12-16 17:49:45.079723] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [128685][T1_HBService][T1][YB42C0A89F89-00062963E023AA14-0-0] [lt=27] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:45.161251] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8F-000629642BDC7AD9-0-0] [lt=30] [DDL] execute ddl like stmt(ret=0, cost=18, ddl_arg=NULL) [2024-12-16 17:49:45.162306] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [128655][T1_StandbySchem][T1][YB42C0A89F89-00062963DD93FAD7-0-0] [lt=36] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:45.249370] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [130219][T1002_FrzInfoDe][T1002][YB42C0A89F89-00062963EAD36923-0-0] [lt=61] get number(ret=0, service_type_=0, range=1, base_id=1734400185244941000, start_id=1734400185244941000, end_id=1734400185244941001) [2024-12-16 17:49:45.254986] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [130219][T1002_FrzInfoDe][T1002][YB42C0A89F89-00062963EAD36923-0-0] [lt=118] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1734400185244941000, v:0}, tenant_id=1002) [2024-12-16 17:49:45.314846] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F80A-0-0] [lt=75] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202081794000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202110029000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:0, operation_type_:0, create_scn:{val:1734359202119679000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:45.317499] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F80A-0-0] [lt=107] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-16 17:49:45.317516] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F80A-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:45.324923] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8A-00062963DFD99B2E-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=24, ddl_arg=NULL) [2024-12-16 17:49:45.387894] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8D-000629641B98C90F-0-0] [lt=22] [DDL] execute ddl like stmt(ret=0, cost=20, ddl_arg=NULL) [2024-12-16 17:49:45.597448] INFO [RS] do_work (ob_common_ls_service.cpp:138) [128656][T1_COMMONLSSe][T1][YB42C0A89F89-00062963E043FA88-0-0] [lt=42] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:45.712896] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F42F-0-0] [lt=25] is primary tenant, no need check readable_scn [2024-12-16 17:49:45.726258] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:345) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F42F-0-0] [lt=23] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1002, ls_id:{id:1}, sync_scn:{val:1734400185479568001, v:0}, readable_scn:{val:1734400185479568001, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:1, config_seq:2}}) [2024-12-16 17:49:45.731231] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F42F-0-0] [lt=37] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.159.137:2882", readable_scn:{val:1734400185479568001, v:0}}, {server:"192.168.159.138:2882", readable_scn:{val:1734400185479568002, v:0}}, {server:"192.168.159.139:2882", readable_scn:{val:1734400185479568002, v:0}}], last_dump_ts_=1734400180526791, force_dump=false) [2024-12-16 17:49:45.731272] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F42F-0-0] [lt=35] is primary tenant, no need check readable_scn [2024-12-16 17:49:45.740733] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [129810][T1002_TeRec][T1002][YB42C0A89F89-00062963E523F42F-0-0] [lt=59] is primary tenant, no need check readable_scn [2024-12-16 17:49:45.753511] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.159.137:2882") [2024-12-16 17:49:45.753555] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=43] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="192.168.159.138:2882") [2024-12-16 17:49:45.753564] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="192.168.159.139:2882") [2024-12-16 17:49:45.753572] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=3, dest="192.168.159.141:2882") [2024-12-16 17:49:45.753578] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=4, dest="192.168.159.142:2882") [2024-12-16 17:49:45.753585] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [129807][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=5, dest="192.168.159.143:2882") [2024-12-16 17:49:45.760968] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [128684][T1_HBService][T1][YB42C0A89F89-00062963E003A9C8-0-0] [lt=48] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:45.783317] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129974][T1001_StandbySc][T1001][YB42C0A89F89-00062963E5D3F95E-0-0] [lt=25] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:45.924094] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8B-00062963DFDFA024-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=19, ddl_arg=NULL) [2024-12-16 17:49:46.000535] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F420-0-0] [lt=19] [COMMON_LS_SERVICE] start to create ls(info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, create_scn={val:1734359202119679000, v:0}) [2024-12-16 17:49:46.016193] WDIAG [RS] do_create_user_ls (ob_common_ls_service.cpp:269) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F420-0-0] [lt=8][errcode=-4672] failed to create user ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, locality_array=[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], create_scn={val:1734359202119679000, v:0}, palf_base_info={prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}, curr_lsn:{lsn:18446744073709551615}}, create_with_palf=false, source_tenant_id=0) [2024-12-16 17:49:46.016241] INFO [RS] do_create_user_ls (ob_common_ls_service.cpp:274) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F420-0-0] [lt=38] [COMMON_LS_SERVICE] end to create ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, cost=15670) [2024-12-16 17:49:46.016253] WDIAG [RS] try_create_ls_ (ob_common_ls_service.cpp:179) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F420-0-0] [lt=12][errcode=-4672] failed to create new ls(ret=-4672, ret="OB_REPLICA_NUM_NOT_ENOUGH", status_info={tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, recovery_stat={tenant_id:1002, ls_id:{id:1003}, sync_scn:{val:1, v:0}, readable_scn:{val:1, v:0}, create_scn:{val:1734359202119679000, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:9223372036854775807, config_seq:-1}}) [2024-12-16 17:49:46.016274] WDIAG [RS] do_work (ob_common_ls_service.cpp:117) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F420-0-0] [lt=17][errcode=0] failed to create ls(ret=0, ret="OB_SUCCESS", tmp_ret=-4672, tmp_ret="OB_REPLICA_NUM_NOT_ENOUGH", user_tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:46.022955] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F420-0-0] [lt=41] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}, {unit_group_id:1002, unit_status:0, ls_group_ids:[]}]) [2024-12-16 17:49:46.027315] INFO [RS] do_work (ob_common_ls_service.cpp:138) [129975][T1001_COMMONLSS][T1001][YB42C0A89F89-00062963EC13F420-0-0] [lt=51] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-16 17:49:46.033016] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [129809][T1002_StandbySc][T1002][YB42C0A89F89-00062963E533F960-0-0] [lt=36] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:46.052344] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [128481][LeaseQueueTh0][T0][YB42C0A89F8E-0006296420107359-0-0] [lt=105] [DDL] execute ddl like stmt(ret=0, cost=16, ddl_arg=NULL) [2024-12-16 17:49:46.163354] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [128655][T1_StandbySchem][T1][YB42C0A89F89-00062963DD93FAD8-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-16 17:49:46.321200] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F80B-0-0] [lt=145] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202081794000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1734359202110029000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"CREATING", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:0, operation_type_:0, create_scn:{val:1734359202119679000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-16 17:49:46.324166] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F80B-0-0] [lt=227] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-16 17:49:46.324188] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [129812][T1002_PLSSer][T1002][YB42C0A89F89-00062963EBE3F80B-0-0] [lt=22] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1734359202408408, tenant_name:"t1", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}})