[2024-06-26 08:50:24.671387] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717DE8-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:24.719866] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CE4-0-0] [lt=15] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363024417582002, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363024417582002, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363024417582002, v:0}}]) [2024-06-26 08:50:24.727439] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:440) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CE4-0-0] [lt=68] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1719363022292438002, v:0}) [2024-06-26 08:50:24.804015] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17DEB-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:25.273784] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217CFC-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:25.275383] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217CFC-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:25.275398] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217CFC-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:25.429946] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:25.429962] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:25.429966] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:25.429972] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=281, return_code_array=[0, 0, 0], broadcast_times=2) [2024-06-26 08:50:25.497510] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C77-0-0] [lt=16] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:25.499312] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317DED-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:25.499615] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C77-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:25.671500] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717DE9-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:25.728846] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CE5-0-0] [lt=38] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363025227182004, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363025227182004, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363025227182004, v:0}}]) [2024-06-26 08:50:25.804068] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17DEC-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:26.277102] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217CFD-0-0] [lt=23] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:26.278439] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217CFD-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:26.278452] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217CFD-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:26.429628] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363024417582002, v:0}, replayable_scn:{val:1719363024417582002, v:0}, standby_scn:{val:1719363024417582002, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363026428765, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363024417582002, v:0}, replayable_scn:{val:1719363024417582002, v:0}, standby_scn:{val:1719363024417582002, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363026428765, ora_rowscn_=1719363025730152000) [2024-06-26 08:50:26.429907] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=25] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:26.429918] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:26.429921] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:26.499398] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317DEE-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:26.503088] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C78-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:26.505076] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C78-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:26.599799] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B74-0-0] [lt=8] start do ObArchiveSchedulerService round(round=29234) [2024-06-26 08:50:26.601625] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B74-0-0] [lt=16][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:50:26.601642] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B74-0-0] [lt=16][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:50:26.601667] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B74-0-0] [lt=6][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:26.601680] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B74-0-0] [lt=13][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:50:26.601687] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B74-0-0] [lt=6][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:26.618562] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=4] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363026428765, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363024417582002, v:0}, replayable_scn:{val:1719363024417582002, v:0}, standby_scn:{val:1719363024417582002, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:50:26.671645] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717DEA-0-0] [lt=56] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:26.734489] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CE6-0-0] [lt=14] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363026228122004, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363026228122004, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363026228122004, v:0}}]) [2024-06-26 08:50:26.804158] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17DED-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:27.280085] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217CFE-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:27.281413] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217CFE-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:27.281426] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217CFE-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:27.404382] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:390) [62463][T1001_FrzInfoDe][T1001][YB420A0A0A34-00061BB2CCF0B01D-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1719363027403689000, start_id=1719363027403689000, end_id=1719363027403689001) [2024-06-26 08:50:27.405405] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:340) [62463][T1001_FrzInfoDe][T1001][YB420A0A0A34-00061BB2CCF0B01D-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1719363027403689000, v:0}, tenant_id=1001) [2024-06-26 08:50:27.429946] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:27.429959] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:27.429962] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:27.499479] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317DEF-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:27.508577] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C79-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:27.510594] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C79-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:27.671728] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717DEB-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:27.740973] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CE7-0-0] [lt=17] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363027294310004, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363027294310004, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363027294310004, v:0}}]) [2024-06-26 08:50:27.804238] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17DEE-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:28.283215] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217CFF-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:28.284540] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217CFF-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:28.284556] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217CFF-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:28.430049] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:28.430068] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:28.430071] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:28.499592] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317DF0-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:28.514048] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C7A-0-0] [lt=16] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:28.515774] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C7A-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:28.601764] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B75-0-0] [lt=9] start do ObArchiveSchedulerService round(round=29235) [2024-06-26 08:50:28.603260] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B75-0-0] [lt=13][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:50:28.603274] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B75-0-0] [lt=13][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:50:28.603296] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B75-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:28.603307] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B75-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:50:28.603313] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B75-0-0] [lt=5][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:28.671813] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717DEC-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:28.747277] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CE8-0-0] [lt=17] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363028230533004, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363028230533004, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363028230533004, v:0}}]) [2024-06-26 08:50:28.749497] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:336) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CE8-0-0] [lt=12] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1002, ls_id:{id:1}, sync_scn:{val:1719363028230533004, v:0}, readable_scn:{val:1719363028230533004, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:5, config_seq:6}}) [2024-06-26 08:50:28.804317] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17DEF-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:29.286311] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D00-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:29.287660] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D00-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:29.287675] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D00-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:29.429948] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363027294310004, v:0}, replayable_scn:{val:1719363027294310004, v:0}, standby_scn:{val:1719363027294310004, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363029429011, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363027294310004, v:0}, replayable_scn:{val:1719363027294310004, v:0}, standby_scn:{val:1719363027294310004, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363029429011, ora_rowscn_=1719363028748594000) [2024-06-26 08:50:29.430224] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=29] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:29.430237] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:29.430241] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:29.430257] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=279, return_code_array=[0, 0, 0], broadcast_times=1) [2024-06-26 08:50:29.499705] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317DF1-0-0] [lt=43] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:29.519165] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C7B-0-0] [lt=16] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:29.520951] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C7B-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:29.620416] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=5] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363029429011, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363027294310004, v:0}, replayable_scn:{val:1719363027294310004, v:0}, standby_scn:{val:1719363027294310004, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:50:29.671895] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717DED-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:29.753243] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CE9-0-0] [lt=15] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363029420493002, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363029420493002, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363029420493002, v:0}}]) [2024-06-26 08:50:29.804396] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17DF0-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:30.289642] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D01-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:30.291245] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D01-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:30.291261] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D01-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:30.430157] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:30.430171] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:30.430175] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:30.499789] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317DF2-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:30.524013] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C7C-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:30.525729] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C7C-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:30.603386] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B76-0-0] [lt=6] start do ObArchiveSchedulerService round(round=29236) [2024-06-26 08:50:30.604854] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B76-0-0] [lt=14][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:50:30.604868] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B76-0-0] [lt=13][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:50:30.604889] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B76-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:30.604902] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B76-0-0] [lt=13][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:50:30.604907] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B76-0-0] [lt=4][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:30.671974] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717DEE-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:30.759491] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CEA-0-0] [lt=16] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363030232155004, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363030232155004, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363030232155004, v:0}}]) [2024-06-26 08:50:30.804476] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17DF1-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:31.297310] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D02-0-0] [lt=34] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:31.301402] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D02-0-0] [lt=190] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:31.301416] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D02-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:31.430324] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:31.430338] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:31.430343] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:31.499895] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317DF3-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:31.530522] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C7D-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:31.533951] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C7D-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:31.672052] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717DEF-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:31.765369] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CEB-0-0] [lt=15] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363031233081004, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363031233081004, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363031233081003, v:0}}]) [2024-06-26 08:50:31.804562] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17DF2-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:32.303105] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D03-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:32.304521] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D03-0-0] [lt=104] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:32.304535] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D03-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:32.430121] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=15] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363030232155004, v:0}, replayable_scn:{val:1719363030232155004, v:0}, standby_scn:{val:1719363030232155004, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363032429148, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363030232155004, v:0}, replayable_scn:{val:1719363030232155004, v:0}, standby_scn:{val:1719363030232155004, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363032429148, ora_rowscn_=1719363031766245000) [2024-06-26 08:50:32.430415] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=22] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:32.430428] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:32.430431] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:32.430436] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=293, return_code_array=[0, 0, 0], broadcast_times=1) [2024-06-26 08:50:32.499975] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317DF4-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:32.537753] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C7E-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:32.539556] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C7E-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:32.604992] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B77-0-0] [lt=4] start do ObArchiveSchedulerService round(round=29237) [2024-06-26 08:50:32.610854] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B77-0-0] [lt=42][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:50:32.610895] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B77-0-0] [lt=39][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:50:32.610985] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B77-0-0] [lt=27][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:32.611020] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B77-0-0] [lt=34][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:50:32.611046] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B77-0-0] [lt=25][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:32.622271] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=5] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363032429148, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363030232155004, v:0}, replayable_scn:{val:1719363030232155004, v:0}, standby_scn:{val:1719363030232155004, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:50:32.672138] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717DF0-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:32.772214] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CEC-0-0] [lt=17] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363032295193004, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363032295193004, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363032295193004, v:0}}]) [2024-06-26 08:50:32.804637] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17DF3-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:33.306239] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D04-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:33.307609] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D04-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:33.307624] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D04-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:33.387688] INFO [STORAGE.TRANS] submit_log_ (ob_id_service.cpp:157) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B01D-0-0] [lt=54] submit log success(service_type_=0, last_id=1719363049933948000, limited_id=1719363049933948000) [2024-06-26 08:50:33.387700] INFO [STORAGE.TRANS] submit_log_with_lock_ (ob_id_service.cpp:65) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B01D-0-0] [lt=11] submit log with lock success(service_type_=0, last_id=1719363049933948000, limited_id=1719363049933948000) [2024-06-26 08:50:33.387704] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:390) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B01D-0-0] [lt=3] get number(ret=0, service_type_=0, range=1, base_id=1719363033386611000, start_id=1719363033386611000, end_id=1719363033386611001) [2024-06-26 08:50:33.387922] INFO [STORAGE.TRANS] submit_log_ (ob_id_service.cpp:109) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B01D-0-0] [lt=4] is logging [2024-06-26 08:50:33.387932] INFO [STORAGE.TRANS] submit_log_with_lock_ (ob_id_service.cpp:57) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B01D-0-0] [lt=9] submit log with lock(ret=-4023, service_type_=0, last_id=1719363049933948000, limited_id=1719363049933948000) [2024-06-26 08:50:33.388406] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:340) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B01D-0-0] [lt=4] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1719363033386611000, v:0}, tenant_id=1002) [2024-06-26 08:50:33.430592] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:33.430607] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:33.430610] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:33.500051] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317DF5-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:33.543070] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C7F-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:33.544989] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C7F-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:33.672221] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717DF1-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:33.777951] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CED-0-0] [lt=16] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363033387674003, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363033387674003, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363033387674003, v:0}}]) [2024-06-26 08:50:33.804717] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17DF4-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:34.309420] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D05-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:34.310933] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D05-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:34.310949] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D05-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:34.430358] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:34.430370] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:34.430374] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:34.500136] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317DF6-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:34.554427] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C80-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:34.558996] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C80-0-0] [lt=54] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:34.611156] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B78-0-0] [lt=41] start do ObArchiveSchedulerService round(round=29238) [2024-06-26 08:50:34.612937] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B78-0-0] [lt=19][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:50:34.612954] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B78-0-0] [lt=16][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:50:34.612980] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B78-0-0] [lt=6][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:34.612990] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B78-0-0] [lt=10][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:50:34.612996] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B78-0-0] [lt=6][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:34.672348] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717DF2-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:34.784572] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CEE-0-0] [lt=17] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363034423039002, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363034423039002, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363034423039002, v:0}}]) [2024-06-26 08:50:34.789573] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:440) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CEE-0-0] [lt=17] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1719363032295193004, v:0}) [2024-06-26 08:50:34.804798] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17DF5-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:35.312715] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D06-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:35.314089] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D06-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:35.314102] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D06-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:35.430159] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363033387674004, v:0}, replayable_scn:{val:1719363033387674004, v:0}, standby_scn:{val:1719363033387674004, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363035429521, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363033387674004, v:0}, replayable_scn:{val:1719363033387674004, v:0}, standby_scn:{val:1719363033387674004, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363035429521, ora_rowscn_=1719363034785055000) [2024-06-26 08:50:35.430455] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=24] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:35.430464] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:35.430467] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:35.500212] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317DF7-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:35.563464] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C81-0-0] [lt=48] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:35.565243] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C81-0-0] [lt=16] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:35.624151] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=5] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363035429521, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363033387674004, v:0}, replayable_scn:{val:1719363033387674004, v:0}, standby_scn:{val:1719363033387674004, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:50:35.672525] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717DF3-0-0] [lt=68] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:35.790926] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CEF-0-0] [lt=13] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363035237973004, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363035237973004, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363035237973004, v:0}}]) [2024-06-26 08:50:35.804880] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17DF6-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:36.316129] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D07-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:36.317755] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D07-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:36.317769] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D07-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:36.430499] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:36.430513] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:36.430520] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:36.430526] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=247, return_code_array=[0, 0, 0], broadcast_times=1) [2024-06-26 08:50:36.500309] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317DF8-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:36.568810] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C82-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:36.570694] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C82-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:36.613070] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B79-0-0] [lt=9] start do ObArchiveSchedulerService round(round=29239) [2024-06-26 08:50:36.614511] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B79-0-0] [lt=14][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:50:36.614525] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B79-0-0] [lt=13][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:50:36.614548] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B79-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:36.614556] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B79-0-0] [lt=8][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:50:36.614561] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B79-0-0] [lt=5][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:36.672645] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717DF4-0-0] [lt=52] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:36.796835] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CF0-0-0] [lt=17] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363036238364004, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363036238364004, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363036238364004, v:0}}]) [2024-06-26 08:50:36.804959] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17DF7-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:37.319549] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D08-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:37.320900] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D08-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:37.320913] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D08-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:37.409119] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:390) [62463][T1001_FrzInfoDe][T1001][YB420A0A0A34-00061BB2CCF0B01E-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1719363037408584000, start_id=1719363037408584000, end_id=1719363037408584001) [2024-06-26 08:50:37.410178] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:340) [62463][T1001_FrzInfoDe][T1001][YB420A0A0A34-00061BB2CCF0B01E-0-0] [lt=18] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1719363037408584000, v:0}, tenant_id=1001) [2024-06-26 08:50:37.410200] INFO [RS.COMPACTION] try_reload (ob_zone_merge_manager.cpp:138) [62463][T1001_FrzInfoDe][T1001][YB420A0A0A34-00061BB2CCF0B01E-0-0] [lt=19] zone_merge_mgr is already loaded(tenant_id=1001, global_merge_info={tenant_id:1001, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1719338404694050000, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1719338404694050000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719338404694050000, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719338404711365, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719338687490915, need_update:false}}, zone_merge_infos=[{tenant_id:1001, zone:"zone1", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1719338404694050000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719338404694050000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719338687492646, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1719338404694050000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719338404719308, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1719338404694050000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1001, zone:"zone2", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1719338404694050000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719338404694050000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719338687494952, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1719338404694050000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719338404720685, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1719338404694050000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1001, zone:"zone3", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1719338404694050000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719338404694050000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719338687496076, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1719338404694050000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719338404721877, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1719338404694050000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}]) [2024-06-26 08:50:37.430694] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:37.430706] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:37.430710] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:37.500423] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317DF9-0-0] [lt=44] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:37.574122] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C83-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:37.575792] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C83-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:37.672724] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717DF5-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:37.803135] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CF1-0-0] [lt=23] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363037297131004, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363037297131004, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363037297131004, v:0}}]) [2024-06-26 08:50:37.805384] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17DF8-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:38.322697] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D09-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:38.324071] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D09-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:38.324082] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D09-0-0] [lt=10] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:38.430470] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363036238364004, v:0}, replayable_scn:{val:1719363036238364004, v:0}, standby_scn:{val:1719363036238364004, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363038429451, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363036238364004, v:0}, replayable_scn:{val:1719363036238364004, v:0}, standby_scn:{val:1719363036238364004, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363038429451, ora_rowscn_=1719363037803714000) [2024-06-26 08:50:38.430778] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=28] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:38.430794] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:38.430798] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:38.500503] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317DFA-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:38.579100] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C84-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:38.581075] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C84-0-0] [lt=29] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:38.614658] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7A-0-0] [lt=4] start do ObArchiveSchedulerService round(round=29240) [2024-06-26 08:50:38.618875] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7A-0-0] [lt=43][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:50:38.618892] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7A-0-0] [lt=16][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:50:38.618917] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7A-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:38.618927] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7A-0-0] [lt=10][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:50:38.618932] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7A-0-0] [lt=5][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:38.626138] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=6] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363038429451, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363036238364004, v:0}, replayable_scn:{val:1719363036238364004, v:0}, standby_scn:{val:1719363036238364004, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:50:38.672822] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717DF6-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:38.805452] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17DF9-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:38.808716] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CF2-0-0] [lt=14] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363038240728004, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363038740832000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363038740832000, v:0}}]) [2024-06-26 08:50:38.811003] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:336) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CF2-0-0] [lt=11] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1002, ls_id:{id:1}, sync_scn:{val:1719363038740832000, v:0}, readable_scn:{val:1719363038740832000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:5, config_seq:6}}) [2024-06-26 08:50:39.326113] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D0A-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:39.327761] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D0A-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:39.327778] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D0A-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:39.430839] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:39.430852] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:39.430856] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:39.430861] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=410, return_code_array=[0, 0, 0], broadcast_times=1) [2024-06-26 08:50:39.500601] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317DFB-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:39.590392] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C85-0-0] [lt=25] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:39.594676] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C85-0-0] [lt=55] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:39.672906] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717DF7-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:39.805555] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17DFA-0-0] [lt=39] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:39.814885] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CF3-0-0] [lt=15] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363039426279002, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363039743499000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363039743499000, v:0}}]) [2024-06-26 08:50:40.329615] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D0B-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:40.330957] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D0B-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:40.330971] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D0B-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:40.430816] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:40.430831] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:40.430834] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:40.500683] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317DFC-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:40.602589] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C86-0-0] [lt=43] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:40.604530] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C86-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:40.619008] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7B-0-0] [lt=9] start do ObArchiveSchedulerService round(round=29241) [2024-06-26 08:50:40.620464] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7B-0-0] [lt=14][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:50:40.620478] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7B-0-0] [lt=13][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:50:40.620501] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7B-0-0] [lt=6][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:40.620509] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7B-0-0] [lt=8][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:50:40.620515] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7B-0-0] [lt=5][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:40.672988] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717DF8-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:40.805643] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17DFB-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:40.821561] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CF4-0-0] [lt=13] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363040242932004, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363040743992000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363040743992000, v:0}}]) [2024-06-26 08:50:41.332761] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D0C-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:41.334122] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D0C-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:41.334134] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D0C-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:41.430704] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363039743499000, v:0}, replayable_scn:{val:1719363039743499000, v:0}, standby_scn:{val:1719363039743499000, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363041429225, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363039743499000, v:0}, replayable_scn:{val:1719363039743499000, v:0}, standby_scn:{val:1719363039743499000, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363041429225, ora_rowscn_=1719363040823993000) [2024-06-26 08:50:41.431050] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=29] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:41.431073] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=22] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:41.431077] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:41.500763] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317DFD-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:41.608301] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C87-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:41.610729] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C87-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:41.628124] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363041429225, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363039743499000, v:0}, replayable_scn:{val:1719363039743499000, v:0}, standby_scn:{val:1719363039743499000, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:50:41.673070] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717DF9-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:41.805727] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17DFC-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:41.831715] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CF5-0-0] [lt=35] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363041244394004, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363041744465000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363041744465000, v:0}}]) [2024-06-26 08:50:42.048830] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:714) [61872][T1001_BackupDat][T1001][YB420A0A0A34-00061BB2CD10A1A7-0-0] [lt=16] [DATA_BACKUP]success get jobs(job_attrs=[]) [2024-06-26 08:50:42.336159] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D0D-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:42.337790] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D0D-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:42.337807] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D0D-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:42.430969] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:42.430983] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:42.430988] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:42.500843] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317DFE-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:42.613721] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C88-0-0] [lt=16] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:42.615481] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C88-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:42.620587] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7C-0-0] [lt=4] start do ObArchiveSchedulerService round(round=29242) [2024-06-26 08:50:42.622321] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7C-0-0] [lt=14][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:50:42.622335] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7C-0-0] [lt=13][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:50:42.622360] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7C-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:42.622369] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7C-0-0] [lt=9][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:50:42.622375] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7C-0-0] [lt=5][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:42.673163] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717DFA-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:42.805810] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17DFD-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:42.837656] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CF6-0-0] [lt=15] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363042745854001, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363042745854001, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363042745854001, v:0}}]) [2024-06-26 08:50:43.339758] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D0E-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:43.341420] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D0E-0-0] [lt=71] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:43.341438] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D0E-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:43.391828] INFO [STORAGE.TRANS] submit_log_ (ob_id_service.cpp:157) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B01E-0-0] [lt=11] submit log success(service_type_=0, last_id=1719363059933948000, limited_id=1719363059933948000) [2024-06-26 08:50:43.391840] INFO [STORAGE.TRANS] submit_log_with_lock_ (ob_id_service.cpp:65) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B01E-0-0] [lt=11] submit log with lock success(service_type_=0, last_id=1719363059933948000, limited_id=1719363059933948000) [2024-06-26 08:50:43.391844] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:390) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B01E-0-0] [lt=3] get number(ret=0, service_type_=0, range=1, base_id=1719363043391243000, start_id=1719363043391243000, end_id=1719363043391243001) [2024-06-26 08:50:43.392063] INFO [STORAGE.TRANS] submit_log_ (ob_id_service.cpp:109) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B01E-0-0] [lt=4] is logging [2024-06-26 08:50:43.392071] INFO [STORAGE.TRANS] submit_log_with_lock_ (ob_id_service.cpp:57) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B01E-0-0] [lt=8] submit log with lock(ret=-4023, service_type_=0, last_id=1719363059933948000, limited_id=1719363059933948000) [2024-06-26 08:50:43.392807] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:340) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B01E-0-0] [lt=5] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1719363043391243000, v:0}, tenant_id=1002) [2024-06-26 08:50:43.392859] INFO [RS.COMPACTION] try_reload (ob_zone_merge_manager.cpp:138) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B01E-0-0] [lt=44] zone_merge_mgr is already loaded(tenant_id=1002, global_merge_info={tenant_id:1002, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1719338400730168000, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1719338400730168000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719338400730168000, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719338400758131, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719338694295419, need_update:false}}, zone_merge_infos=[{tenant_id:1002, zone:"zone1", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1719338400730168000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719338400730168000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719338694297127, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1719338400730168000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719338400783878, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1719338400730168000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1002, zone:"zone2", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1719338400730168000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719338400730168000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719338694299395, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1719338400730168000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719338400788109, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1719338400730168000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1002, zone:"zone3", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1719338400730168000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719338400730168000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719338694300496, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1719338400730168000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719338400789744, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1719338400730168000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}]) [2024-06-26 08:50:43.431149] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:43.431163] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:43.431166] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:43.431172] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=413, return_code_array=[0, 0, 0], broadcast_times=2) [2024-06-26 08:50:43.500928] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317DFF-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:43.620504] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C89-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:43.625091] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C89-0-0] [lt=35] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:43.673236] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717DFB-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:43.805892] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17DFE-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:43.843555] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CF7-0-0] [lt=15] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363043746589000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363043746589000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363043746589000, v:0}}]) [2024-06-26 08:50:44.343154] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D0F-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:44.344522] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D0F-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:44.344535] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D0F-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:44.430794] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363042745854001, v:0}, replayable_scn:{val:1719363042745854001, v:0}, standby_scn:{val:1719363042745854001, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363044429182, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363042745854001, v:0}, replayable_scn:{val:1719363042745854001, v:0}, standby_scn:{val:1719363042745854001, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363044429182, ora_rowscn_=1719363043844047000) [2024-06-26 08:50:44.431051] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=25] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:44.431073] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=22] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:44.431076] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:44.501008] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E00-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:44.622455] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7D-0-0] [lt=5] start do ObArchiveSchedulerService round(round=29243) [2024-06-26 08:50:44.623929] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7D-0-0] [lt=17][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:50:44.623945] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7D-0-0] [lt=15][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:50:44.623967] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7D-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:44.623978] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7D-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:50:44.623983] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7D-0-0] [lt=5][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:44.628228] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C8A-0-0] [lt=25] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:44.629891] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C8A-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:44.630117] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=4] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363044429182, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363042745854001, v:0}, replayable_scn:{val:1719363042745854001, v:0}, standby_scn:{val:1719363042745854001, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:50:44.673383] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717DFC-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:44.805990] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17DFF-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:44.850018] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CF8-0-0] [lt=33] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363044748034000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363044748034000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363044748034000, v:0}}]) [2024-06-26 08:50:44.854657] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:440) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CF8-0-0] [lt=14] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1719363042745854001, v:0}) [2024-06-26 08:50:45.346217] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D10-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:45.347554] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D10-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:45.347568] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D10-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:45.431154] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:45.431167] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:45.431170] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:45.501087] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E01-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:45.637614] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C8B-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:45.641672] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C8B-0-0] [lt=56] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:45.673461] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717DFD-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:45.806071] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E00-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:45.855916] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CF9-0-0] [lt=14] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363045749318000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363045749318000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363045749318000, v:0}}]) [2024-06-26 08:50:46.349229] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D11-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:46.350630] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D11-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:46.350642] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D11-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:46.431496] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:46.431509] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:46.431513] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:46.431518] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=442, return_code_array=[0, 0, 0], broadcast_times=1) [2024-06-26 08:50:46.501198] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E02-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:46.624064] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7E-0-0] [lt=5] start do ObArchiveSchedulerService round(round=29244) [2024-06-26 08:50:46.625650] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7E-0-0] [lt=13][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:50:46.625663] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7E-0-0] [lt=13][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:50:46.625686] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7E-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:46.625698] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7E-0-0] [lt=12][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:50:46.625703] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7E-0-0] [lt=4][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:46.645514] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C8C-0-0] [lt=32] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:46.647543] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C8C-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:46.673564] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717DFE-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:46.806194] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E01-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:46.862012] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CFA-0-0] [lt=15] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363046750180000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363046750180000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363046750180000, v:0}}]) [2024-06-26 08:50:47.352314] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D12-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:47.353648] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D12-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:47.353663] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D12-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:47.416106] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:390) [62463][T1001_FrzInfoDe][T1001][YB420A0A0A34-00061BB2CCF0B01F-0-0] [lt=76] get number(ret=0, service_type_=0, range=1, base_id=1719363047416040000, start_id=1719363047416040000, end_id=1719363047416040001) [2024-06-26 08:50:47.417550] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:340) [62463][T1001_FrzInfoDe][T1001][YB420A0A0A34-00061BB2CCF0B01F-0-0] [lt=69] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1719363047416040000, v:0}, tenant_id=1001) [2024-06-26 08:50:47.431089] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363045749318000, v:0}, replayable_scn:{val:1719363045749318000, v:0}, standby_scn:{val:1719363045749318000, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363047429845, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363045749318000, v:0}, replayable_scn:{val:1719363045749318000, v:0}, standby_scn:{val:1719363045749318000, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363047429845, ora_rowscn_=1719363046863128000) [2024-06-26 08:50:47.431466] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=28] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:47.431476] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:47.431480] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:47.501318] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E03-0-0] [lt=52] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:47.632116] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=5] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363047429845, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363045749318000, v:0}, replayable_scn:{val:1719363045749318000, v:0}, standby_scn:{val:1719363045749318000, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:50:47.650745] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C8D-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:47.652494] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C8D-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:47.673674] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717DFF-0-0] [lt=44] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:47.806316] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E02-0-0] [lt=46] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:47.869081] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CFB-0-0] [lt=19] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363047751544000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363047751544000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363047751544000, v:0}}]) [2024-06-26 08:50:48.355324] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D13-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:48.356645] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D13-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:48.356660] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D13-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:48.431311] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:48.431323] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:48.431327] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:48.501424] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E04-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:48.625777] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7F-0-0] [lt=7] start do ObArchiveSchedulerService round(round=29245) [2024-06-26 08:50:48.627259] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7F-0-0] [lt=14][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:50:48.627273] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7F-0-0] [lt=13][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:50:48.627295] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7F-0-0] [lt=4][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:48.627306] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7F-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:50:48.627311] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B7F-0-0] [lt=5][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:48.655939] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C8E-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:48.657551] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C8E-0-0] [lt=16] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:48.673758] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E00-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:48.806400] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E03-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:48.875285] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CFC-0-0] [lt=16] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363048752391000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363048752391000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363048752391000, v:0}}]) [2024-06-26 08:50:48.877490] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:336) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CFC-0-0] [lt=11] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1002, ls_id:{id:1}, sync_scn:{val:1719363048752391000, v:0}, readable_scn:{val:1719363048752391000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:5, config_seq:6}}) [2024-06-26 08:50:49.362722] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D14-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:49.368205] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D14-0-0] [lt=242] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:49.368252] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D14-0-0] [lt=45] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:49.431609] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:49.431625] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:49.431630] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:49.431636] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=405, return_code_array=[0, 0, 0], broadcast_times=1) [2024-06-26 08:50:49.501540] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E05-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:49.660978] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C8F-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:49.662892] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C8F-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:49.673843] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E01-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:49.806482] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E04-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:49.881490] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CFD-0-0] [lt=13] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363049754719000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363049754719000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363049754719000, v:0}}]) [2024-06-26 08:50:50.369947] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D15-0-0] [lt=82] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:50.371332] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D15-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:50.371347] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D15-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:50.431267] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363048752391000, v:0}, replayable_scn:{val:1719363048752391000, v:0}, standby_scn:{val:1719363048752391000, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363050430441, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363048752391000, v:0}, replayable_scn:{val:1719363048752391000, v:0}, standby_scn:{val:1719363048752391000, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363050430441, ora_rowscn_=1719363049882659000) [2024-06-26 08:50:50.431560] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=26] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:50.431570] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:50.431573] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:50.501652] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E06-0-0] [lt=43] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:50.627387] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B80-0-0] [lt=7] start do ObArchiveSchedulerService round(round=29246) [2024-06-26 08:50:50.628895] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B80-0-0] [lt=14][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:50:50.628911] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B80-0-0] [lt=15][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:50:50.628933] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B80-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:50.628945] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B80-0-0] [lt=12][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:50:50.628950] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B80-0-0] [lt=5][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:50.634149] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=5] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363050430441, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363048752391000, v:0}, replayable_scn:{val:1719363048752391000, v:0}, standby_scn:{val:1719363048752391000, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:50:50.667563] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C90-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:50.670649] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C90-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:50.673930] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E02-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:50.806562] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E05-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:50.888467] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CFE-0-0] [lt=17] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363050755044000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363050755044000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363050755044000, v:0}}]) [2024-06-26 08:50:51.373294] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D16-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:51.374902] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D16-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:51.374919] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D16-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:51.431624] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:51.431640] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:51.431644] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:51.501736] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E07-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:51.673989] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C91-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:51.674005] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E03-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:51.676262] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C91-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:51.806644] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E06-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:51.894642] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817CFF-0-0] [lt=14] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363051756354000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363051756354000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363051756354000, v:0}}]) [2024-06-26 08:50:52.376589] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D17-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:52.378001] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D17-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:52.378015] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D17-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:52.433415] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=42] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:52.433453] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=37] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:52.433470] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:52.433495] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=18] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=411, return_code_array=[0, 0, 0], broadcast_times=1) [2024-06-26 08:50:52.501819] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E08-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:52.629024] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B81-0-0] [lt=5] start do ObArchiveSchedulerService round(round=29247) [2024-06-26 08:50:52.630839] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B81-0-0] [lt=15][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:50:52.630856] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B81-0-0] [lt=16][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:50:52.630882] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B81-0-0] [lt=6][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:52.630895] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B81-0-0] [lt=12][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:50:52.630902] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B81-0-0] [lt=7][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:52.674137] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E04-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:52.679538] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C92-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:52.681976] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C92-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:52.806728] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E07-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:52.900599] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D00-0-0] [lt=5] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363052757983000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363052757983000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363052757983000, v:0}}]) [2024-06-26 08:50:53.379675] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D18-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:53.381026] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D18-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:53.381041] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D18-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:53.402014] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:390) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B01F-0-0] [lt=227] get number(ret=0, service_type_=0, range=1, base_id=1719363053401950000, start_id=1719363053401950000, end_id=1719363053401950001) [2024-06-26 08:50:53.403644] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:340) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B01F-0-0] [lt=72] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1719363053401950000, v:0}, tenant_id=1002) [2024-06-26 08:50:53.431404] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=26] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363051756354000, v:0}, replayable_scn:{val:1719363051756354000, v:0}, standby_scn:{val:1719363051756354000, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363053430574, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363051756354000, v:0}, replayable_scn:{val:1719363051756354000, v:0}, standby_scn:{val:1719363051756354000, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363053430574, ora_rowscn_=1719363052901075000) [2024-06-26 08:50:53.431681] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=26] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:53.431691] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:53.431695] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:53.501937] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E09-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:53.636140] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=6] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363053430574, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363051756354000, v:0}, replayable_scn:{val:1719363051756354000, v:0}, standby_scn:{val:1719363051756354000, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:50:53.674294] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E05-0-0] [lt=51] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:53.685323] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C93-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:53.687138] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C93-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:53.806810] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E08-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:53.907115] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D01-0-0] [lt=16] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363053758759000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363053758759000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363053758759000, v:0}}]) [2024-06-26 08:50:54.382994] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D19-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:54.384577] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D19-0-0] [lt=68] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:54.384595] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D19-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:54.431750] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:54.431764] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:54.431767] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:54.502065] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E0A-0-0] [lt=44] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:54.630973] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B82-0-0] [lt=8] start do ObArchiveSchedulerService round(round=29248) [2024-06-26 08:50:54.632758] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B82-0-0] [lt=15][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:50:54.632774] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B82-0-0] [lt=15][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:50:54.632802] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B82-0-0] [lt=6][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:54.632815] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B82-0-0] [lt=13][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:50:54.632821] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B82-0-0] [lt=6][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:54.674407] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E06-0-0] [lt=43] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:54.690112] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C94-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:54.692612] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C94-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:54.806887] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E09-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:54.913768] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D02-0-0] [lt=15] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363054759249000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363054759249000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363054759249000, v:0}}]) [2024-06-26 08:50:54.918431] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:440) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D02-0-0] [lt=19] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1719363052757983000, v:0}) [2024-06-26 08:50:55.386425] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D1A-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:55.387811] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D1A-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:55.387826] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D1A-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:55.432066] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:55.432086] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=20] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:55.432090] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:55.502150] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E0B-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:55.674493] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E07-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:55.695859] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C95-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:55.697627] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C95-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:55.806967] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E0A-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:55.919792] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D03-0-0] [lt=11] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363055760597000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363055760597001, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363055760597001, v:0}}]) [2024-06-26 08:50:56.389879] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D1B-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:56.391527] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D1B-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:56.391544] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D1B-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:56.431721] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363054759249001, v:0}, replayable_scn:{val:1719363054759249001, v:0}, standby_scn:{val:1719363054759249001, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363056430862, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363054759249001, v:0}, replayable_scn:{val:1719363054759249001, v:0}, standby_scn:{val:1719363054759249001, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363056430862, ora_rowscn_=1719363055920873000) [2024-06-26 08:50:56.432138] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=41] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:56.432150] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:56.432154] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:56.432159] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=396, return_code_array=[0, 0, 0], broadcast_times=1) [2024-06-26 08:50:56.502237] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E0C-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:56.632905] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B83-0-0] [lt=5] start do ObArchiveSchedulerService round(round=29249) [2024-06-26 08:50:56.638078] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=13] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363056430862, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363054759249001, v:0}, replayable_scn:{val:1719363054759249001, v:0}, standby_scn:{val:1719363054759249001, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:50:56.638665] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B83-0-0] [lt=39][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:50:56.638703] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B83-0-0] [lt=35][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:50:56.638787] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B83-0-0] [lt=23][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:56.638823] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B83-0-0] [lt=35][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:50:56.638842] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B83-0-0] [lt=18][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:56.674579] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E08-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:56.700637] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C96-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:56.703765] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C96-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:56.807059] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E0B-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:56.928442] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D04-0-0] [lt=18] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363056762119000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363056762119001, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363056762119001, v:0}}]) [2024-06-26 08:50:57.393539] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D1C-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:57.395163] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D1C-0-0] [lt=67] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:57.395180] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D1C-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:57.422376] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:390) [62463][T1001_FrzInfoDe][T1001][YB420A0A0A34-00061BB2CCF0B020-0-0] [lt=40] get number(ret=0, service_type_=0, range=1, base_id=1719363057422058000, start_id=1719363057422058000, end_id=1719363057422058001) [2024-06-26 08:50:57.423351] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:340) [62463][T1001_FrzInfoDe][T1001][YB420A0A0A34-00061BB2CCF0B020-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1719363057422058000, v:0}, tenant_id=1001) [2024-06-26 08:50:57.431812] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:57.431826] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:57.431830] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:57.502321] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E0D-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:57.674673] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E09-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:57.706794] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C97-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:57.708471] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C97-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:57.807160] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E0C-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:57.934706] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D05-0-0] [lt=12] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363057763840000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363057763840001, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363057763840001, v:0}}]) [2024-06-26 08:50:58.396904] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D1D-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:58.398280] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D1D-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:58.398293] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D1D-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:58.432091] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:58.432109] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:58.432112] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:58.502489] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E0E-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:58.638925] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B84-0-0] [lt=16] start do ObArchiveSchedulerService round(round=29250) [2024-06-26 08:50:58.640495] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B84-0-0] [lt=14][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:50:58.640509] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B84-0-0] [lt=13][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:50:58.640532] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B84-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:58.640543] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B84-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:50:58.640548] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B84-0-0] [lt=4][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:50:58.674756] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E0A-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:58.711614] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C98-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:58.713552] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C98-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:58.807243] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E0D-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:58.941189] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D06-0-0] [lt=20] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363058764238000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363058764238001, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363058764238001, v:0}}]) [2024-06-26 08:50:58.943681] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:336) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D06-0-0] [lt=14] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1002, ls_id:{id:1}, sync_scn:{val:1719363058764238001, v:0}, readable_scn:{val:1719363058764238001, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:5, config_seq:6}}) [2024-06-26 08:50:59.399978] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D1E-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:59.402403] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D1E-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:50:59.402447] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D1E-0-0] [lt=44] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:50:59.433441] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363057763840001, v:0}, replayable_scn:{val:1719363057763840001, v:0}, standby_scn:{val:1719363057763840001, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363059430446, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363057763840001, v:0}, replayable_scn:{val:1719363057763840001, v:0}, standby_scn:{val:1719363057763840001, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363059430446, ora_rowscn_=1719363058942843000) [2024-06-26 08:50:59.434091] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=81] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:50:59.434109] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:50:59.434113] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:50:59.434121] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=598, return_code_array=[0, 0, 0], broadcast_times=1) [2024-06-26 08:50:59.502605] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E0F-0-0] [lt=43] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:59.640064] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=8] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363059430446, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363057763840001, v:0}, replayable_scn:{val:1719363057763840001, v:0}, standby_scn:{val:1719363057763840001, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:50:59.674836] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E0B-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:59.717115] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C99-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:50:59.719219] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C99-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:50:59.807325] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E0E-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:50:59.947732] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D07-0-0] [lt=19] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363059765476001, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363059765476001, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363059765476001, v:0}}]) [2024-06-26 08:51:00.404353] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D1F-0-0] [lt=47] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:00.405746] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D1F-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:00.405758] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D1F-0-0] [lt=11] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:00.432184] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:00.432198] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:00.432202] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:00.502691] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E10-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:00.640581] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B85-0-0] [lt=7] start do ObArchiveSchedulerService round(round=29251) [2024-06-26 08:51:00.642414] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B85-0-0] [lt=16][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:51:00.642430] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B85-0-0] [lt=15][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:51:00.642474] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B85-0-0] [lt=6][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:00.642492] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B85-0-0] [lt=18][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:51:00.642498] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B85-0-0] [lt=5][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:00.674919] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E0C-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:00.722327] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C9A-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:00.723936] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C9A-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:00.807402] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E0F-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:00.954012] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D08-0-0] [lt=12] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363060765880001, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363060765880001, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363060765880001, v:0}}]) [2024-06-26 08:51:01.407561] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D20-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:01.408959] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D20-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:01.408970] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D20-0-0] [lt=10] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:01.432257] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:01.432272] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:01.432276] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:01.502802] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E11-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:01.674998] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E0D-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:01.728812] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C9B-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:01.731967] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C9B-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:01.807485] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E10-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:01.959958] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D09-0-0] [lt=15] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363061767022001, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363061767022001, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363061767022001, v:0}}]) [2024-06-26 08:51:02.412107] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D21-0-0] [lt=23] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:02.417658] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D21-0-0] [lt=205] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:02.417705] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D21-0-0] [lt=45] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:02.432238] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:02.432251] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:02.432255] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:02.502883] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E12-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:02.641918] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=6] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363062431197, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363060765880001, v:0}, replayable_scn:{val:1719363060765880001, v:0}, standby_scn:{val:1719363060765880001, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:51:02.642567] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B86-0-0] [lt=5] start do ObArchiveSchedulerService round(round=29252) [2024-06-26 08:51:02.644035] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B86-0-0] [lt=14][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:51:02.644050] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B86-0-0] [lt=14][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:51:02.644079] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B86-0-0] [lt=11][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:02.644090] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B86-0-0] [lt=10][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:51:02.644094] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B86-0-0] [lt=4][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:02.675074] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E0E-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:02.735553] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C9C-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:02.737535] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C9C-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:02.807574] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E11-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:02.966930] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D0A-0-0] [lt=17] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363062768349001, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363062768349001, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363062768349001, v:0}}]) [2024-06-26 08:51:03.410588] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:390) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B020-0-0] [lt=38] get number(ret=0, service_type_=0, range=1, base_id=1719363063410181000, start_id=1719363063410181000, end_id=1719363063410181001) [2024-06-26 08:51:03.411516] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:340) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B020-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1719363063410181000, v:0}, tenant_id=1002) [2024-06-26 08:51:03.419346] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D22-0-0] [lt=81] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:03.420722] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D22-0-0] [lt=52] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:03.420736] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D22-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:03.432007] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363061767022001, v:0}, replayable_scn:{val:1719363061767022001, v:0}, standby_scn:{val:1719363061767022001, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363063431404, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363061767022001, v:0}, replayable_scn:{val:1719363061767022001, v:0}, standby_scn:{val:1719363061767022001, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363063431404, ora_rowscn_=1719363062967676000) [2024-06-26 08:51:03.432287] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=22] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:03.432297] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:03.432300] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:03.432305] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=275, return_code_array=[0, 0, 0], broadcast_times=2) [2024-06-26 08:51:03.502963] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E13-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:03.675166] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E0F-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:03.740659] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C9D-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:03.742518] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C9D-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:03.807655] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E12-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:03.974853] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D0B-0-0] [lt=16] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363063768953001, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363063768953001, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363063768953001, v:0}}]) [2024-06-26 08:51:04.427202] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D23-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:04.430891] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D23-0-0] [lt=181] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:04.430906] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D23-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:04.433798] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:04.433837] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=38] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:04.433850] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:04.503046] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E14-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:04.644162] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B87-0-0] [lt=7] start do ObArchiveSchedulerService round(round=29253) [2024-06-26 08:51:04.645694] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B87-0-0] [lt=13][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:51:04.645706] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B87-0-0] [lt=11][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:51:04.645729] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B87-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:04.645740] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B87-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:51:04.645745] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B87-0-0] [lt=5][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:04.675251] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E10-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:04.746031] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C9E-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:04.747900] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C9E-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:04.807737] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E13-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:04.981102] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D0C-0-0] [lt=14] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363064770150001, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363064770150001, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363064770150001, v:0}}]) [2024-06-26 08:51:04.985510] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:440) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D0C-0-0] [lt=16] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1719363062768349001, v:0}) [2024-06-26 08:51:05.432510] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:05.432531] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=20] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:05.432536] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:05.436012] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D24-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:05.437645] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D24-0-0] [lt=69] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:05.437669] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D24-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:05.503133] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E15-0-0] [lt=22] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:05.643950] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=5] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363065430687, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363063768953001, v:0}, replayable_scn:{val:1719363063768953001, v:0}, standby_scn:{val:1719363063768953001, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:51:05.675332] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E11-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:05.751075] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C9F-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:05.752858] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517C9F-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:05.807814] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E14-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:05.986898] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D0D-0-0] [lt=13] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363065771489001, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363065771489001, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363065771489001, v:0}}]) [2024-06-26 08:51:06.432210] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363064770150001, v:0}, replayable_scn:{val:1719363064770150001, v:0}, standby_scn:{val:1719363064770150001, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363066430793, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363064770150001, v:0}, replayable_scn:{val:1719363064770150001, v:0}, standby_scn:{val:1719363064770150001, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363066430793, ora_rowscn_=1719363065987807000) [2024-06-26 08:51:06.432514] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=24] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:06.432524] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:06.432528] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:06.432533] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=298, return_code_array=[0, 0, 0], broadcast_times=2) [2024-06-26 08:51:06.439616] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D25-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:06.441258] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D25-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:06.441274] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D25-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:06.443481] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:126) [61869][T1001_REST_SER][T1001][YB420A0A0A34-00061BB2CD00A880-0-0] [lt=12] [RESTORE] restore scheduler quit [2024-06-26 08:51:06.444031] INFO [SHARE] get_all_recover_table_job (ob_recover_table_persist_helper.cpp:199) [61869][T1001_REST_SER][T1001][YB420A0A0A34-00061BB2CD00A880-0-0] [lt=12] get targets(ret=0, jobs=[], exec_tenant_id=1001, sql=select * from __all_recover_table_job) [2024-06-26 08:51:06.444499] INFO [SHARE] get_all_import_table_jobs (ob_import_table_persist_helper.cpp:109) [61869][T1001_REST_SER][T1001][YB420A0A0A34-00061BB2CD00A881-0-0] [lt=17] get targets(ret=0, jobs=[], exec_tenant_id=1001, sql=select * from __all_import_table_job) [2024-06-26 08:51:06.444514] WDIAG [SHARE.SCHEMA] revert (ob_schema_mgr_cache.cpp:122) [61869][T1001_REST_SER][T1001][YB420A0A0A34-00061BB2CD00A882-0-0] [lt=11][errcode=0] long time to hold one guard(schema_mgr=0x2abf03d15260, tenant_id=1, version=1719298380350128, cur_timestamp=1719363066444511, ref_timestamp=1719363006442359, lbt()="0x17a30451 0x1523bc9f 0x5f9b859 0x5f9bdc0 0x5ff915f 0xac59643 0x8f5ffef 0x62d7da4 0x17b107dc 0x17b1be21 0x17b1814e 0x2abe35ce2dc5 0x2abe35fee73d") [2024-06-26 08:51:06.444597] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:95) [61869][T1001_REST_SER][T1001][YB420A0A0A34-00061BB2CD00A883-0-0] [lt=10] [RESTORE] restore scheduler start [2024-06-26 08:51:06.444603] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:102) [61869][T1001_REST_SER][T1001][YB420A0A0A34-00061BB2CD00A884-0-0] [lt=6] [RESTORE] try process restore job [2024-06-26 08:51:06.445011] INFO [COMMON] get_jobs (ob_physical_restore_table_operator.cpp:416) [61869][T1001_REST_SER][T1001][YB420A0A0A34-00061BB2CD00A884-0-0] [lt=2] [RESTORE] get restore jobs(ret=0, job_cnt=0) [2024-06-26 08:51:06.503213] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E16-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:06.645808] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B88-0-0] [lt=7] start do ObArchiveSchedulerService round(round=29254) [2024-06-26 08:51:06.651499] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B88-0-0] [lt=36][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:51:06.651545] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B88-0-0] [lt=44][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:51:06.651634] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B88-0-0] [lt=26][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:06.651667] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B88-0-0] [lt=32][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:51:06.651692] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B88-0-0] [lt=24][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:06.675413] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E12-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:06.755922] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CA0-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:06.757562] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CA0-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:06.807895] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E15-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:06.993512] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D0E-0-0] [lt=18] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363066772115001, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363066772115001, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363066772115001, v:0}}]) [2024-06-26 08:51:07.427336] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:390) [62463][T1001_FrzInfoDe][T1001][YB420A0A0A34-00061BB2CCF0B021-0-0] [lt=15] get number(ret=0, service_type_=0, range=1, base_id=1719363067427188000, start_id=1719363067427188000, end_id=1719363067427188001) [2024-06-26 08:51:07.428337] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:340) [62463][T1001_FrzInfoDe][T1001][YB420A0A0A34-00061BB2CCF0B021-0-0] [lt=19] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1719363067427188000, v:0}, tenant_id=1001) [2024-06-26 08:51:07.432570] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:07.432585] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:07.432588] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:07.443162] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D26-0-0] [lt=22] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:07.444807] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D26-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:07.444823] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D26-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:07.503298] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E17-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:07.675465] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E13-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:07.761176] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CA1-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:07.763050] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CA1-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:07.807981] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E16-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:07.998834] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D0F-0-0] [lt=15] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363067773512001, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363067773512001, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363067773512001, v:0}}]) [2024-06-26 08:51:08.432902] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:08.432917] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:08.432922] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:08.446791] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D27-0-0] [lt=23] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:08.448421] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D27-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:08.448442] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D27-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:08.503383] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E18-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:08.646064] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=5] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363068430727, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363066772115001, v:0}, replayable_scn:{val:1719363066772115001, v:0}, standby_scn:{val:1719363066772115001, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:51:08.651790] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B89-0-0] [lt=22] start do ObArchiveSchedulerService round(round=29255) [2024-06-26 08:51:08.653258] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B89-0-0] [lt=10][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:51:08.653271] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B89-0-0] [lt=12][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:51:08.653293] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B89-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:08.653305] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B89-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:51:08.653309] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B89-0-0] [lt=4][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:08.675546] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E14-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:08.766390] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CA2-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:08.768316] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CA2-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:08.808089] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E17-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:09.005437] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D10-0-0] [lt=18] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363068775712001, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363068775712001, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363068775712001, v:0}}]) [2024-06-26 08:51:09.007504] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:336) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D10-0-0] [lt=12] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1002, ls_id:{id:1}, sync_scn:{val:1719363068775712001, v:0}, readable_scn:{val:1719363068775712001, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:5, config_seq:6}}) [2024-06-26 08:51:09.432407] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363067773512001, v:0}, replayable_scn:{val:1719363067773512001, v:0}, standby_scn:{val:1719363067773512001, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363069431644, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363067773512001, v:0}, replayable_scn:{val:1719363067773512001, v:0}, standby_scn:{val:1719363067773512001, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363069431644, ora_rowscn_=1719363069006218000) [2024-06-26 08:51:09.432702] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=24] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:09.432713] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:09.432716] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:09.432721] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=290, return_code_array=[0, 0, 0], broadcast_times=1) [2024-06-26 08:51:09.450306] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D28-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:09.451907] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D28-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:09.451923] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D28-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:09.503463] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E19-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:09.675631] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E15-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:09.771637] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CA3-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:09.773416] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CA3-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:09.808141] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E18-0-0] [lt=21] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:10.011412] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D11-0-0] [lt=16] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363069776602001, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363069776602001, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363069776602001, v:0}}]) [2024-06-26 08:51:10.433012] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:10.433028] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:10.433032] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:10.453532] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D29-0-0] [lt=23] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:10.454893] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D29-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:10.454913] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D29-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:10.503559] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E1A-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:10.653385] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8A-0-0] [lt=7] start do ObArchiveSchedulerService round(round=29256) [2024-06-26 08:51:10.655129] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8A-0-0] [lt=15][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:51:10.655144] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8A-0-0] [lt=14][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:51:10.655171] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8A-0-0] [lt=6][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:10.655183] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8A-0-0] [lt=12][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:51:10.655190] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8A-0-0] [lt=7][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:10.675745] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E16-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:10.776675] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CA4-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:10.778542] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CA4-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:10.808201] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E19-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:11.017744] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D12-0-0] [lt=18] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363070777644001, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363070777644001, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363070777644001, v:0}}]) [2024-06-26 08:51:11.432886] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:11.432902] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:11.432905] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:11.459153] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D2A-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:11.463977] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D2A-0-0] [lt=139] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:11.464022] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D2A-0-0] [lt=43] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:11.503643] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E1B-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:11.648096] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=4] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363071431551, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363069776602001, v:0}, replayable_scn:{val:1719363069776602001, v:0}, standby_scn:{val:1719363069776602001, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:51:11.675872] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E17-0-0] [lt=49] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:11.782041] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CA5-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:11.782496] INFO [RS] try_update_primary_ip_list (ob_common_ls_service.cpp:360) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CA5-0-0] [lt=12] log restore source is empty, just skip(ret=-4018, user_tenant_id=1002) [2024-06-26 08:51:11.784338] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CA5-0-0] [lt=4] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:11.808284] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E1A-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:12.023714] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D13-0-0] [lt=15] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363071778977001, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363071778977001, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363071778977001, v:0}}]) [2024-06-26 08:51:12.049580] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:714) [61872][T1001_BackupDat][T1001][YB420A0A0A34-00061BB2CD10A1A8-0-0] [lt=14] [DATA_BACKUP]success get jobs(job_attrs=[]) [2024-06-26 08:51:12.091163] INFO [RS] dump_statistics (ob_backup_task_scheduler.cpp:245) [61871][T1001_BACKUP_SC][T1001][YB420A0A0A34-00061BB2CD209943-0-0] [lt=16] backup scheduler task statistics(waiting_task_cnt=0, executing_task_cnt=0) [2024-06-26 08:51:12.432609] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363070777644001, v:0}, replayable_scn:{val:1719363070777644001, v:0}, standby_scn:{val:1719363070777644001, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363072431813, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363070777644001, v:0}, replayable_scn:{val:1719363070777644001, v:0}, standby_scn:{val:1719363070777644001, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363072431813, ora_rowscn_=1719363072024324000) [2024-06-26 08:51:12.432894] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=26] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:12.432906] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:12.432909] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:12.432923] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=11] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=286, return_code_array=[0, 0, 0], broadcast_times=2) [2024-06-26 08:51:12.465735] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D2B-0-0] [lt=100] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:12.467165] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D2B-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:12.467180] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D2B-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:12.503731] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E1C-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:12.655271] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8B-0-0] [lt=5] start do ObArchiveSchedulerService round(round=29257) [2024-06-26 08:51:12.656778] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8B-0-0] [lt=15][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:51:12.656791] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8B-0-0] [lt=12][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:51:12.656814] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8B-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:12.656826] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8B-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:51:12.656831] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8B-0-0] [lt=5][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:12.675988] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E18-0-0] [lt=43] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:12.787463] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CA6-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:12.789196] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CA6-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:12.808365] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E1B-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:13.030419] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D14-0-0] [lt=14] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363072926342000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363072926342000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363072926342000, v:0}}]) [2024-06-26 08:51:13.415159] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:390) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B021-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1719363073414432000, start_id=1719363073414432000, end_id=1719363073414432001) [2024-06-26 08:51:13.416129] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:340) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B021-0-0] [lt=17] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1719363073414432000, v:0}, tenant_id=1002) [2024-06-26 08:51:13.433005] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:13.433020] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:13.433024] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:13.469197] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D2C-0-0] [lt=22] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:13.470784] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D2C-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:13.470800] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D2C-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:13.503810] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E1D-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:13.676076] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E19-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:13.792331] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CA7-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:13.794005] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CA7-0-0] [lt=16] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:13.808440] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E1C-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:14.036112] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D15-0-0] [lt=10] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363073920531000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363074024791000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363074024791000, v:0}}]) [2024-06-26 08:51:14.433019] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:14.433033] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:14.433037] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:14.472615] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D2D-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:14.474016] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D2D-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:14.474028] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D2D-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:14.503900] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E1E-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:14.649897] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=5] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363074431555, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363073023402000, v:0}, replayable_scn:{val:1719363073023402000, v:0}, standby_scn:{val:1719363073023402000, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:51:14.656898] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8C-0-0] [lt=4] start do ObArchiveSchedulerService round(round=29258) [2024-06-26 08:51:14.658614] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8C-0-0] [lt=13][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:51:14.658629] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8C-0-0] [lt=14][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:51:14.658655] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8C-0-0] [lt=6][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:14.658666] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8C-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:51:14.658674] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8C-0-0] [lt=7][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:14.676155] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E1A-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:14.797557] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CA8-0-0] [lt=26] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:14.799402] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CA8-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:14.808540] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E1D-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:15.042397] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D16-0-0] [lt=17] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363074782063001, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363074782063002, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363074782063002, v:0}}]) [2024-06-26 08:51:15.047152] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:440) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D16-0-0] [lt=17] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1719363073023402000, v:0}) [2024-06-26 08:51:15.432993] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363074035424000, v:0}, replayable_scn:{val:1719363074035424000, v:0}, standby_scn:{val:1719363074035424000, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363075431192, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363074035424000, v:0}, replayable_scn:{val:1719363074035424000, v:0}, standby_scn:{val:1719363074035424000, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363075431192, ora_rowscn_=1719363075044058000) [2024-06-26 08:51:15.433258] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=32] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:15.433271] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:15.433275] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:15.475684] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D2E-0-0] [lt=22] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:15.477004] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D2E-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:15.477018] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D2E-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:15.503991] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E1F-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:15.676240] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E1B-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:15.803014] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CA9-0-0] [lt=16] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:15.804789] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CA9-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:15.808624] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E1E-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:16.048361] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D17-0-0] [lt=14] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363075782309001, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363075782309002, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363075782309002, v:0}}]) [2024-06-26 08:51:16.433245] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:16.433263] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:16.433267] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:16.433273] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=275, return_code_array=[0, 0, 0], broadcast_times=1) [2024-06-26 08:51:16.478685] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D2F-0-0] [lt=22] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:16.480061] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D2F-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:16.480075] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D2F-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:16.504075] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E20-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:16.658744] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8D-0-0] [lt=5] start do ObArchiveSchedulerService round(round=29259) [2024-06-26 08:51:16.660240] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8D-0-0] [lt=14][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:51:16.660254] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8D-0-0] [lt=14][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:51:16.660278] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8D-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:16.660290] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8D-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:51:16.660295] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8D-0-0] [lt=4][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:16.676327] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E1C-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:16.808043] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CAA-0-0] [lt=16] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:16.808706] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E1F-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:16.809789] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CAA-0-0] [lt=26] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:17.053882] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D18-0-0] [lt=14] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363076783674001, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363076783674002, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363076783674002, v:0}}]) [2024-06-26 08:51:17.432419] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:390) [62463][T1001_FrzInfoDe][T1001][YB420A0A0A34-00061BB2CCF0B022-0-0] [lt=15] get number(ret=0, service_type_=0, range=1, base_id=1719363077431557000, start_id=1719363077431557000, end_id=1719363077431557001) [2024-06-26 08:51:17.433584] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:340) [62463][T1001_FrzInfoDe][T1001][YB420A0A0A34-00061BB2CCF0B022-0-0] [lt=20] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1719363077431557000, v:0}, tenant_id=1001) [2024-06-26 08:51:17.433938] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:17.433955] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:17.433959] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:17.481902] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D30-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:17.483272] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D30-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:17.483288] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D30-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:17.504151] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E21-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:17.651894] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=4] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363077431557, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363075782309002, v:0}, replayable_scn:{val:1719363075782309002, v:0}, standby_scn:{val:1719363075782309002, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:51:17.676417] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E1D-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:17.808778] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E20-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:17.813069] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CAB-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:17.814891] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CAB-0-0] [lt=16] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:18.059809] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D19-0-0] [lt=16] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363077784631002, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363077784631002, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363077784631002, v:0}}]) [2024-06-26 08:51:18.433117] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363076783674002, v:0}, replayable_scn:{val:1719363076783674002, v:0}, standby_scn:{val:1719363076783674002, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363078431387, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363076783674002, v:0}, replayable_scn:{val:1719363076783674002, v:0}, standby_scn:{val:1719363076783674002, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363078431387, ora_rowscn_=1719363078060914000) [2024-06-26 08:51:18.433400] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=24] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:18.433412] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:18.433415] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:18.485224] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D31-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:18.486806] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D31-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:18.486823] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D31-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:18.504226] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E22-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:18.660366] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8E-0-0] [lt=6] start do ObArchiveSchedulerService round(round=29260) [2024-06-26 08:51:18.661831] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8E-0-0] [lt=13][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:51:18.661845] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8E-0-0] [lt=14][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:51:18.661877] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8E-0-0] [lt=15][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:18.661889] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8E-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:51:18.661893] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8E-0-0] [lt=4][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:18.676498] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E1E-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:18.808860] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E21-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:18.818111] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CAC-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:18.820233] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CAC-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:19.065406] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D1A-0-0] [lt=16] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363078785154002, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363078785154002, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363078785154002, v:0}}]) [2024-06-26 08:51:19.067662] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:336) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D1A-0-0] [lt=14] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1002, ls_id:{id:1}, sync_scn:{val:1719363078785154002, v:0}, readable_scn:{val:1719363078785154002, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:5, config_seq:6}}) [2024-06-26 08:51:19.435094] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:19.435134] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=39] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:19.435149] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:19.435175] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=19] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=393, return_code_array=[0, 0, 0], broadcast_times=2) [2024-06-26 08:51:19.488748] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D32-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:19.490388] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D32-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:19.490404] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D32-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:19.504310] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E23-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:19.676588] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E1F-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:19.808938] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E22-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:19.823481] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CAD-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:19.825154] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CAD-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:20.071949] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D1B-0-0] [lt=19] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363079786821002, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363079786821002, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363079786821002, v:0}}]) [2024-06-26 08:51:20.433475] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=28] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:20.433488] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:20.433492] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:20.492135] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D33-0-0] [lt=22] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:20.493514] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D33-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:20.493530] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D33-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:20.504389] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E24-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:20.653757] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=12] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363080431942, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363078785154002, v:0}, replayable_scn:{val:1719363078785154002, v:0}, standby_scn:{val:1719363078785154002, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:51:20.661960] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8F-0-0] [lt=5] start do ObArchiveSchedulerService round(round=29261) [2024-06-26 08:51:20.663567] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8F-0-0] [lt=12][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:51:20.663580] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8F-0-0] [lt=13][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:51:20.663605] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8F-0-0] [lt=6][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:20.663617] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8F-0-0] [lt=12][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:51:20.663622] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B8F-0-0] [lt=4][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:20.676667] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E20-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:20.809015] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E23-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:20.828225] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CAE-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:20.830192] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CAE-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:21.077710] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D1C-0-0] [lt=15] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363080787673002, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363080787673002, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363080787673002, v:0}}]) [2024-06-26 08:51:21.433332] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363079786821002, v:0}, replayable_scn:{val:1719363079786821002, v:0}, standby_scn:{val:1719363079786821002, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363081432127, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363079786821002, v:0}, replayable_scn:{val:1719363079786821002, v:0}, standby_scn:{val:1719363079786821002, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363081432127, ora_rowscn_=1719363081078774000) [2024-06-26 08:51:21.433752] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=25] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:21.433765] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:21.433768] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:21.495489] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D34-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:21.497064] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D34-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:21.497087] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D34-0-0] [lt=23] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:21.504462] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E25-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:21.676744] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E21-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:21.809098] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E24-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:21.833414] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CAF-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:21.835567] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CAF-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:22.083658] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D1D-0-0] [lt=15] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363081788897002, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363081788897002, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363081788897002, v:0}}]) [2024-06-26 08:51:22.433819] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:22.433836] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:22.433843] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:22.498755] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D35-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:22.500104] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D35-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:22.500116] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D35-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:22.504542] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E26-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:22.663691] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B90-0-0] [lt=4] start do ObArchiveSchedulerService round(round=29262) [2024-06-26 08:51:22.665126] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B90-0-0] [lt=13][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:51:22.665140] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B90-0-0] [lt=13][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:51:22.665163] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B90-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:22.665174] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B90-0-0] [lt=10][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:51:22.665180] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B90-0-0] [lt=5][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:22.676827] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E22-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:22.809179] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E25-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:22.839254] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CB0-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:22.841140] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CB0-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:23.089147] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D1E-0-0] [lt=13] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363082789228002, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363082789228002, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363082789228002, v:0}}]) [2024-06-26 08:51:23.420022] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:390) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B022-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1719363083419673000, start_id=1719363083419673000, end_id=1719363083419673001) [2024-06-26 08:51:23.420977] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:340) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B022-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1719363083419673000, v:0}, tenant_id=1002) [2024-06-26 08:51:23.433676] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:23.433691] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:23.433695] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:23.433700] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=306, return_code_array=[0, 0, 0], broadcast_times=1) [2024-06-26 08:51:23.501848] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D36-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:23.503208] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D36-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:23.503221] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D36-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:23.504617] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E27-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:23.655604] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=4] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363083432600, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363081788897002, v:0}, replayable_scn:{val:1719363081788897002, v:0}, standby_scn:{val:1719363081788897002, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:51:23.676906] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E23-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:23.809256] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E26-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:23.844295] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CB1-0-0] [lt=16] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:23.846091] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CB1-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:24.095563] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D1F-0-0] [lt=16] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363083790468002, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363083790468002, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363083790468002, v:0}}]) [2024-06-26 08:51:24.433543] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363082789228002, v:0}, replayable_scn:{val:1719363082789228002, v:0}, standby_scn:{val:1719363082789228002, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363084432674, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363082789228002, v:0}, replayable_scn:{val:1719363082789228002, v:0}, standby_scn:{val:1719363082789228002, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363084432674, ora_rowscn_=1719363084096468000) [2024-06-26 08:51:24.433866] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=26] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:24.433876] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:24.433880] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:24.504694] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E28-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:24.504894] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D37-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:24.506223] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D37-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:24.506238] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D37-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:24.665284] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B91-0-0] [lt=5] start do ObArchiveSchedulerService round(round=29263) [2024-06-26 08:51:24.666966] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B91-0-0] [lt=42][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:51:24.666983] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B91-0-0] [lt=17][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:51:24.667011] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B91-0-0] [lt=6][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:24.667023] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B91-0-0] [lt=12][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:51:24.667030] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B91-0-0] [lt=6][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:24.676988] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E24-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:24.809336] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E27-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:24.849014] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CB2-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:24.850731] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CB2-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:25.101314] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D20-0-0] [lt=17] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363084791069002, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363084791069003, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363084791069003, v:0}}]) [2024-06-26 08:51:25.105916] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:440) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D20-0-0] [lt=17] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1719363082789228002, v:0}) [2024-06-26 08:51:25.434130] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:25.434148] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:25.434152] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:25.504767] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E29-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:25.507921] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D38-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:25.509263] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D38-0-0] [lt=51] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:25.509278] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D38-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:25.677086] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E25-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:25.809427] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E28-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:25.853936] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CB3-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:25.855770] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CB3-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:26.107247] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D21-0-0] [lt=13] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363085792754002, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363085792754002, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363085792754002, v:0}}]) [2024-06-26 08:51:26.434023] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:26.434039] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:26.434043] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:26.434048] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=378, return_code_array=[0, 0, 0], broadcast_times=1) [2024-06-26 08:51:26.504849] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E2A-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:26.511000] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D39-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:26.512352] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D39-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:26.512367] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D39-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:26.657501] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=4] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363086432766, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363084791069003, v:0}, replayable_scn:{val:1719363084791069003, v:0}, standby_scn:{val:1719363084791069003, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:51:26.667109] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B92-0-0] [lt=7] start do ObArchiveSchedulerService round(round=29264) [2024-06-26 08:51:26.668546] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B92-0-0] [lt=13][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:51:26.668559] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B92-0-0] [lt=13][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:51:26.668584] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B92-0-0] [lt=6][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:26.668595] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B92-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:51:26.668600] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B92-0-0] [lt=5][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:26.677170] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E26-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:26.809509] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E29-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:26.859094] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CB4-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:26.860961] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CB4-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:27.113017] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D22-0-0] [lt=16] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363086793516002, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363086793516002, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363086793516002, v:0}}]) [2024-06-26 08:51:27.433858] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=11] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363085792754002, v:0}, replayable_scn:{val:1719363085792754002, v:0}, standby_scn:{val:1719363085792754002, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363087432370, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363085792754002, v:0}, replayable_scn:{val:1719363085792754002, v:0}, standby_scn:{val:1719363085792754002, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363087432370, ora_rowscn_=1719363087114522000) [2024-06-26 08:51:27.434506] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=34] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:27.434524] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:27.434528] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:27.437421] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:390) [62463][T1001_FrzInfoDe][T1001][YB420A0A0A34-00061BB2CCF0B023-0-0] [lt=21] get number(ret=0, service_type_=0, range=1, base_id=1719363087436605000, start_id=1719363087436605000, end_id=1719363087436605001) [2024-06-26 08:51:27.438354] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:340) [62463][T1001_FrzInfoDe][T1001][YB420A0A0A34-00061BB2CCF0B023-0-0] [lt=17] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1719363087436605000, v:0}, tenant_id=1001) [2024-06-26 08:51:27.504933] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E2B-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:27.514073] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D3A-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:27.515418] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D3A-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:27.515430] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D3A-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:27.677257] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E27-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:27.809586] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E2A-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:27.863883] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CB5-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:27.865509] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CB5-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:28.119258] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D23-0-0] [lt=19] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363087794329002, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363087794329002, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363087794329002, v:0}}]) [2024-06-26 08:51:28.435961] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:28.436005] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=42] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:28.436019] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:28.505020] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E2C-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:28.517131] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D3B-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:28.518461] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D3B-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:28.518476] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D3B-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:28.668670] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B93-0-0] [lt=7] start do ObArchiveSchedulerService round(round=29265) [2024-06-26 08:51:28.670418] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B93-0-0] [lt=15][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:51:28.670432] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B93-0-0] [lt=13][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:51:28.670464] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B93-0-0] [lt=6][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:28.670473] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B93-0-0] [lt=9][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:51:28.670479] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B93-0-0] [lt=5][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:28.677349] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E28-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:28.809669] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E2B-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:28.868767] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CB6-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:28.870457] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CB6-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:29.125648] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D24-0-0] [lt=17] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363088795258002, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363088795258002, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363088795258002, v:0}}]) [2024-06-26 08:51:29.127836] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:336) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D24-0-0] [lt=11] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1002, ls_id:{id:1}, sync_scn:{val:1719363088795258002, v:0}, readable_scn:{val:1719363088795258002, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:5, config_seq:6}}) [2024-06-26 08:51:29.434302] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:29.434315] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:29.434319] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:29.505102] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E2D-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:29.520247] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D3C-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:29.521588] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D3C-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:29.521603] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D3C-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:29.659336] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=5] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363089432304, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363087794329002, v:0}, replayable_scn:{val:1719363087794329002, v:0}, standby_scn:{val:1719363087794329002, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:51:29.677456] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E29-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:29.809795] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E2C-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:29.873925] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CB7-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:29.875878] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CB7-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:30.131963] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D25-0-0] [lt=16] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363089795928002, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363089795928002, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363089795928002, v:0}}]) [2024-06-26 08:51:30.433949] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363088795258002, v:0}, replayable_scn:{val:1719363088795258002, v:0}, standby_scn:{val:1719363088795258002, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363090432253, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363088795258002, v:0}, replayable_scn:{val:1719363088795258002, v:0}, standby_scn:{val:1719363088795258002, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363090432253, ora_rowscn_=1719363090132741000) [2024-06-26 08:51:30.434304] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=27] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:30.434318] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:30.434320] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:30.434326] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=350, return_code_array=[0, 0, 0], broadcast_times=2) [2024-06-26 08:51:30.505166] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E2E-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:30.523427] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D3D-0-0] [lt=22] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:30.524768] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D3D-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:30.524784] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D3D-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:30.670563] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B94-0-0] [lt=5] start do ObArchiveSchedulerService round(round=29266) [2024-06-26 08:51:30.672033] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B94-0-0] [lt=14][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:51:30.672046] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B94-0-0] [lt=12][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:51:30.672075] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B94-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:30.672087] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B94-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:51:30.672092] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B94-0-0] [lt=4][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:30.677536] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E2A-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:30.809873] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E2D-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:30.879084] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CB8-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:30.880753] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CB8-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:31.137787] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D26-0-0] [lt=15] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363090796467002, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363090796467002, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363090796467002, v:0}}]) [2024-06-26 08:51:31.436170] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:31.436189] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:31.436194] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:31.505249] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E2F-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:31.526423] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D3E-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:31.527769] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D3E-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:31.527785] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D3E-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:31.677618] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E2B-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:31.809967] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E2E-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:31.885221] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CB9-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:31.888522] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CB9-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:32.143872] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D27-0-0] [lt=16] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363091797616002, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363091797616003, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363091797616003, v:0}}]) [2024-06-26 08:51:32.436340] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:32.436383] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=41] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:32.436398] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:32.505329] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E30-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:32.529463] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D3F-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:32.530814] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D3F-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:32.530825] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D3F-0-0] [lt=11] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:32.661193] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=5] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363092433275, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363090796467003, v:0}, replayable_scn:{val:1719363090796467003, v:0}, standby_scn:{val:1719363090796467003, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:51:32.672387] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B95-0-0] [lt=6] start do ObArchiveSchedulerService round(round=29267) [2024-06-26 08:51:32.673870] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B95-0-0] [lt=14][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:51:32.673881] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B95-0-0] [lt=11][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:51:32.673903] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B95-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:32.673914] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B95-0-0] [lt=10][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:51:32.673919] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B95-0-0] [lt=4][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:32.677700] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E2C-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:32.810094] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E2F-0-0] [lt=51] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:32.891826] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CBA-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:32.893486] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CBA-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:33.150417] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D28-0-0] [lt=16] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363092798321002, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363092798321003, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363092798321003, v:0}}]) [2024-06-26 08:51:33.424453] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:390) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B023-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1719363093424258000, start_id=1719363093424258000, end_id=1719363093424258001) [2024-06-26 08:51:33.425500] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:340) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B023-0-0] [lt=20] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1719363093424258000, v:0}, tenant_id=1002) [2024-06-26 08:51:33.434235] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363091797616003, v:0}, replayable_scn:{val:1719363091797616003, v:0}, standby_scn:{val:1719363091797616003, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363093432748, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363091797616003, v:0}, replayable_scn:{val:1719363091797616003, v:0}, standby_scn:{val:1719363091797616003, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363093432748, ora_rowscn_=1719363093150921000) [2024-06-26 08:51:33.434477] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=28] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:33.434489] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:33.434493] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:33.434502] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=238, return_code_array=[0, 0, 0], broadcast_times=1) [2024-06-26 08:51:33.505407] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E31-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:33.532642] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D40-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:33.533988] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D40-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:33.534002] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D40-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:33.677775] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E2D-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:33.810241] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E30-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:33.896655] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CBB-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:33.898295] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CBB-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:34.156315] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D29-0-0] [lt=17] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363093800099002, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363093800099003, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363093800099003, v:0}}]) [2024-06-26 08:51:34.434787] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:34.434804] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:34.434816] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:34.505495] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E32-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:34.535698] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D41-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:34.537019] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D41-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:34.537032] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D41-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:34.673991] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B96-0-0] [lt=6] start do ObArchiveSchedulerService round(round=29268) [2024-06-26 08:51:34.675444] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B96-0-0] [lt=14][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:51:34.675457] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B96-0-0] [lt=12][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:51:34.675481] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B96-0-0] [lt=6][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:34.675492] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B96-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:51:34.675497] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B96-0-0] [lt=4][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:34.678165] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E2E-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:34.810319] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E31-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:34.902824] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CBC-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:34.904589] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CBC-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:35.161941] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D2A-0-0] [lt=15] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363094800691003, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363094800691004, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363094800691004, v:0}}]) [2024-06-26 08:51:35.166631] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:440) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D2A-0-0] [lt=14] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1719363092798321003, v:0}) [2024-06-26 08:51:35.436449] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:35.436492] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=42] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:35.436507] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:35.505571] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E33-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:35.538706] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D42-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:35.540050] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D42-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:35.540070] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D42-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:35.663038] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=4] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363095432956, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363093800099003, v:0}, replayable_scn:{val:1719363093800099003, v:0}, standby_scn:{val:1719363093800099003, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:51:35.678276] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E2F-0-0] [lt=43] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:35.810402] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E32-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:35.907911] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CBD-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:35.909590] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CBD-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:36.167917] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D2B-0-0] [lt=13] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363095802061003, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363095802061003, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363095802061003, v:0}}]) [2024-06-26 08:51:36.434412] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=11] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363094800691004, v:0}, replayable_scn:{val:1719363094800691004, v:0}, standby_scn:{val:1719363094800691003, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363096433743, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363094800691004, v:0}, replayable_scn:{val:1719363094800691004, v:0}, standby_scn:{val:1719363094800691003, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363096433743, ora_rowscn_=1719363096168484000) [2024-06-26 08:51:36.434786] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=27] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:36.434797] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:36.434800] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:36.434807] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=368, return_code_array=[0, 0, 0], broadcast_times=1) [2024-06-26 08:51:36.505667] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E34-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:36.541689] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D43-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:36.543038] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D43-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:36.543059] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D43-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:36.675572] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B97-0-0] [lt=6] start do ObArchiveSchedulerService round(round=29269) [2024-06-26 08:51:36.676978] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B97-0-0] [lt=14][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:51:36.676994] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B97-0-0] [lt=15][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:51:36.677016] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B97-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:36.677027] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B97-0-0] [lt=10][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:51:36.677031] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B97-0-0] [lt=4][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:36.678362] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E30-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:36.810484] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E33-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:36.912779] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CBE-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:36.914530] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CBE-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:37.173942] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D2C-0-0] [lt=14] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363096802970003, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363096802970003, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363096802970003, v:0}}]) [2024-06-26 08:51:37.436899] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:37.436943] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=43] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:37.436958] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:37.441958] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:390) [62463][T1001_FrzInfoDe][T1001][YB420A0A0A34-00061BB2CCF0B024-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1719363097441014000, start_id=1719363097441014000, end_id=1719363097441014001) [2024-06-26 08:51:37.443286] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:340) [62463][T1001_FrzInfoDe][T1001][YB420A0A0A34-00061BB2CCF0B024-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1719363097441014000, v:0}, tenant_id=1001) [2024-06-26 08:51:37.505781] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E35-0-0] [lt=44] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:37.544945] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D44-0-0] [lt=22] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:37.546313] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D44-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:37.546329] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D44-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:37.678441] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E31-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:37.810563] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E34-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:37.917795] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CBF-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:37.919745] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CBF-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:38.179635] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D2D-0-0] [lt=17] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363097803629003, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363097803629003, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363097803629003, v:0}}]) [2024-06-26 08:51:38.434923] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:38.434938] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:38.434941] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:38.505859] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E36-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:38.547950] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D45-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:38.549294] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D45-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:38.549307] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D45-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:38.664932] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=5] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363098433243, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363096802970003, v:0}, replayable_scn:{val:1719363096802970003, v:0}, standby_scn:{val:1719363096802970003, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:51:38.677104] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B98-0-0] [lt=7] start do ObArchiveSchedulerService round(round=29270) [2024-06-26 08:51:38.678523] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E32-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:38.678554] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B98-0-0] [lt=14][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:51:38.678569] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B98-0-0] [lt=14][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:51:38.678591] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B98-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:38.678601] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B98-0-0] [lt=10][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:51:38.678606] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B98-0-0] [lt=4][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:38.810667] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E35-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:38.923049] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CC0-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:38.924801] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CC0-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:39.185600] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D2E-0-0] [lt=15] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363098804718003, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363098804718003, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363098804718003, v:0}}]) [2024-06-26 08:51:39.187674] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:336) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D2E-0-0] [lt=12] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1002, ls_id:{id:1}, sync_scn:{val:1719363098804718003, v:0}, readable_scn:{val:1719363098804718003, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:5, config_seq:6}}) [2024-06-26 08:51:39.434747] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363097803629003, v:0}, replayable_scn:{val:1719363097803629003, v:0}, standby_scn:{val:1719363097803629003, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363099433578, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363097803629003, v:0}, replayable_scn:{val:1719363097803629003, v:0}, standby_scn:{val:1719363097803629003, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363099433578, ora_rowscn_=1719363099185964000) [2024-06-26 08:51:39.435074] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=28] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:39.435087] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:39.435091] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:39.505957] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E37-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:39.550875] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D46-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:39.552266] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D46-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:39.552279] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D46-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:39.678604] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E33-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:39.810769] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E36-0-0] [lt=36] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:39.927976] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CC1-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:39.929679] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CC1-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:40.191705] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D2F-0-0] [lt=23] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363099806302003, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363099806302003, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363099806302003, v:0}}]) [2024-06-26 08:51:40.436830] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:40.436880] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=48] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:40.436895] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:40.436913] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=551, return_code_array=[0, 0, 0], broadcast_times=1) [2024-06-26 08:51:40.506065] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E38-0-0] [lt=40] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:40.554146] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D47-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:40.555480] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D47-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:40.555494] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D47-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:40.678677] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B99-0-0] [lt=6] start do ObArchiveSchedulerService round(round=29271) [2024-06-26 08:51:40.678684] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E34-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:40.680146] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B99-0-0] [lt=12][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:51:40.680160] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B99-0-0] [lt=13][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:51:40.680183] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B99-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:40.680194] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B99-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:51:40.680199] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B99-0-0] [lt=5][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:40.810847] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E37-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:40.933197] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CC2-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:40.934900] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CC2-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:41.197294] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D30-0-0] [lt=15] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363100807290003, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363100807290003, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363100807290003, v:0}}]) [2024-06-26 08:51:41.435207] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=28] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:41.435226] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:41.435230] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:41.506180] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E39-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:41.557156] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D48-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:41.558598] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D48-0-0] [lt=51] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:41.558613] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D48-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:41.666831] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=5] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363101434065, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363099806302003, v:0}, replayable_scn:{val:1719363099806302003, v:0}, standby_scn:{val:1719363099806302003, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:51:41.678766] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E35-0-0] [lt=9] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:41.810922] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E38-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:41.939398] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CC3-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:41.942866] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CC3-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:42.050247] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:714) [61872][T1001_BackupDat][T1001][YB420A0A0A34-00061BB2CD10A1A9-0-0] [lt=14] [DATA_BACKUP]success get jobs(job_attrs=[]) [2024-06-26 08:51:42.204100] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D31-0-0] [lt=12] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363101808436003, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363101808436003, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363101808436003, v:0}}]) [2024-06-26 08:51:42.434897] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363100807290003, v:0}, replayable_scn:{val:1719363100807290003, v:0}, standby_scn:{val:1719363100807290003, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363102433480, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363100807290003, v:0}, replayable_scn:{val:1719363100807290003, v:0}, standby_scn:{val:1719363100807290003, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363102433480, ora_rowscn_=1719363102205206000) [2024-06-26 08:51:42.435172] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=28] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:42.435185] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:42.435188] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:42.506321] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E3A-0-0] [lt=41] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:42.560199] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D49-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:42.561552] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D49-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:42.561563] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D49-0-0] [lt=11] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:42.678858] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E36-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:42.680271] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9A-0-0] [lt=7] start do ObArchiveSchedulerService round(round=29272) [2024-06-26 08:51:42.681738] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9A-0-0] [lt=14][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:51:42.681752] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9A-0-0] [lt=13][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:51:42.681774] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9A-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:42.681785] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9A-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:51:42.681790] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9A-0-0] [lt=5][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:42.811018] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E39-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:42.946011] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CC4-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:42.947763] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CC4-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:43.210792] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D32-0-0] [lt=17] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363102809702003, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363102809702003, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363102809702003, v:0}}]) [2024-06-26 08:51:43.429400] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:390) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B024-0-0] [lt=15] get number(ret=0, service_type_=0, range=1, base_id=1719363103428854000, start_id=1719363103428854000, end_id=1719363103428854001) [2024-06-26 08:51:43.431595] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:340) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B024-0-0] [lt=57] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1719363103428854000, v:0}, tenant_id=1002) [2024-06-26 08:51:43.435317] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:43.435333] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:43.435338] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:43.506451] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E3B-0-0] [lt=66] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:43.563492] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D4A-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:43.564870] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D4A-0-0] [lt=65] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:43.564884] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D4A-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:43.678940] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E37-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:43.811114] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E3A-0-0] [lt=26] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:43.951157] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CC5-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:43.952861] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CC5-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:44.216857] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D33-0-0] [lt=17] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363103812315003, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363103812315003, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363103812315003, v:0}}]) [2024-06-26 08:51:44.436985] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:44.437003] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:44.437007] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:44.437013] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=384, return_code_array=[0, 0, 0], broadcast_times=1) [2024-06-26 08:51:44.506566] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E3C-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:44.566526] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D4B-0-0] [lt=22] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:44.567884] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D4B-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:44.567898] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D4B-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:44.668807] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=4] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363104433776, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363102809702003, v:0}, replayable_scn:{val:1719363102809702003, v:0}, standby_scn:{val:1719363102809702003, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:51:44.679026] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E38-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:44.681864] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9B-0-0] [lt=7] start do ObArchiveSchedulerService round(round=29273) [2024-06-26 08:51:44.683330] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9B-0-0] [lt=13][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:51:44.683343] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9B-0-0] [lt=13][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:51:44.683367] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9B-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:44.683379] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9B-0-0] [lt=12][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:51:44.683384] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9B-0-0] [lt=4][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:44.811192] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E3B-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:44.957690] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CC6-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:44.960747] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CC6-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:45.222466] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D34-0-0] [lt=14] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363104813191003, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363104813191003, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363104813191003, v:0}}]) [2024-06-26 08:51:45.226711] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:440) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D34-0-0] [lt=21] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1719363102809702003, v:0}) [2024-06-26 08:51:45.436762] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363103812315003, v:0}, replayable_scn:{val:1719363103812315003, v:0}, standby_scn:{val:1719363103812315003, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363105434026, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363103812315003, v:0}, replayable_scn:{val:1719363103812315003, v:0}, standby_scn:{val:1719363103812315003, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363105434026, ora_rowscn_=1719363105223410000) [2024-06-26 08:51:45.437274] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=91] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:45.437305] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=29] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:45.437325] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:45.506718] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E3D-0-0] [lt=43] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:45.569576] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D4C-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:45.570938] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D4C-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:45.570955] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D4C-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:45.679107] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E39-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:45.811272] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E3C-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:45.964012] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CC7-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:45.965829] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CC7-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:46.227939] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D35-0-0] [lt=12] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363105814337003, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363105814337003, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363105814337003, v:0}}]) [2024-06-26 08:51:46.435969] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:46.435987] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:46.435990] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:46.506829] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E3E-0-0] [lt=43] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:46.572560] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D4D-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:46.573951] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D4D-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:46.573964] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D4D-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:46.679192] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E3A-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:46.683467] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9C-0-0] [lt=7] start do ObArchiveSchedulerService round(round=29274) [2024-06-26 08:51:46.684875] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9C-0-0] [lt=10][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:51:46.684889] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9C-0-0] [lt=13][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:51:46.684910] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9C-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:46.684921] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9C-0-0] [lt=10][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:51:46.684925] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9C-0-0] [lt=4][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:46.811351] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E3D-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:46.969459] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CC8-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:46.971258] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CC8-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:47.235189] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D36-0-0] [lt=17] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363106815705003, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363106815705003, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363106815705003, v:0}}]) [2024-06-26 08:51:47.435559] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:47.435574] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:47.435578] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:47.447008] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:390) [62463][T1001_FrzInfoDe][T1001][YB420A0A0A34-00061BB2CCF0B025-0-0] [lt=15] get number(ret=0, service_type_=0, range=1, base_id=1719363107446890000, start_id=1719363107446890000, end_id=1719363107446890001) [2024-06-26 08:51:47.448004] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:340) [62463][T1001_FrzInfoDe][T1001][YB420A0A0A34-00061BB2CCF0B025-0-0] [lt=17] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1719363107446890000, v:0}, tenant_id=1001) [2024-06-26 08:51:47.450161] INFO [STORAGE.TRANS] alloc (ob_trans_factory.cpp:141) [62463][T1001_FrzInfoDe][T1001][YB420A0A0A34-00061BB2CCF0B025-0-0] [lt=16] ObTransCtx statistics(active_coord_ctx_count=0, active_part_ctx_count=1, total_release_part_ctx_count=109) [2024-06-26 08:51:47.506909] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E3F-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:47.575662] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D4E-0-0] [lt=22] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:47.577038] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D4E-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:47.577059] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D4E-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:47.670759] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=4] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363107434152, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363105814337003, v:0}, replayable_scn:{val:1719363105814337003, v:0}, standby_scn:{val:1719363105814337003, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:51:47.679276] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E3B-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:47.811426] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E3E-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:47.974725] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CC9-0-0] [lt=16] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:47.976501] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CC9-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:48.241318] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D37-0-0] [lt=14] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363107816457003, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363107816457003, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363107816457003, v:0}}]) [2024-06-26 08:51:48.435616] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:48.435631] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:48.435634] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:48.435643] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=319, return_code_array=[0, 0, 0], broadcast_times=1) [2024-06-26 08:51:48.507003] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E40-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:48.579171] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D4F-0-0] [lt=22] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:48.580814] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D4F-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:48.580830] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D4F-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:48.679355] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E3C-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:48.684999] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9D-0-0] [lt=7] start do ObArchiveSchedulerService round(round=29275) [2024-06-26 08:51:48.686426] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9D-0-0] [lt=12][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:51:48.686440] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9D-0-0] [lt=13][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:51:48.686462] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9D-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:48.686474] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9D-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:51:48.686482] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9D-0-0] [lt=7][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:48.811507] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E3F-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:48.979972] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CCA-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:48.981734] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CCA-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:49.247182] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D38-0-0] [lt=15] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363108817691003, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363108817691003, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363108817691003, v:0}}]) [2024-06-26 08:51:49.249500] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:336) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D38-0-0] [lt=13] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1002, ls_id:{id:1}, sync_scn:{val:1719363108817691003, v:0}, readable_scn:{val:1719363108817691003, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:5, config_seq:6}}) [2024-06-26 08:51:49.437073] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363107816457003, v:0}, replayable_scn:{val:1719363107816457003, v:0}, standby_scn:{val:1719363107816457003, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363109434287, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363107816457003, v:0}, replayable_scn:{val:1719363107816457003, v:0}, standby_scn:{val:1719363107816457003, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363109434287, ora_rowscn_=1719363109247707000) [2024-06-26 08:51:49.437728] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=86] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:49.437759] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=30] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:49.437780] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=20] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:49.507085] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E41-0-0] [lt=43] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:49.582765] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D50-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:49.584416] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D50-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:49.584431] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D50-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:49.679465] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E3D-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:49.811586] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E40-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:49.991234] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CCB-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:49.995926] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CCB-0-0] [lt=61] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:50.253910] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D39-0-0] [lt=16] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363109818565003, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363109818565004, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363109818565004, v:0}}]) [2024-06-26 08:51:50.437691] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:50.437734] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=41] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:50.437748] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:50.507196] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E42-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:50.586389] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D51-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:50.588050] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D51-0-0] [lt=67] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:50.588073] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D51-0-0] [lt=23] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:50.672603] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=5] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363110433951, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363108817691003, v:0}, replayable_scn:{val:1719363108817691003, v:0}, standby_scn:{val:1719363108817691003, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:51:50.679546] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E3E-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:50.686561] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9E-0-0] [lt=7] start do ObArchiveSchedulerService round(round=29276) [2024-06-26 08:51:50.687956] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9E-0-0] [lt=14][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:51:50.687970] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9E-0-0] [lt=14][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:51:50.687992] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9E-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:50.688003] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9E-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:51:50.688008] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9E-0-0] [lt=5][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:50.811661] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E41-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:51.000578] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CCC-0-0] [lt=49] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:51.003615] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CCC-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:51.259464] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D3A-0-0] [lt=15] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363110819405003, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363110819405004, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363110819405004, v:0}}]) [2024-06-26 08:51:51.435821] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:51.435836] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:51.435839] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:51.435844] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=299, return_code_array=[0, 0, 0], broadcast_times=2) [2024-06-26 08:51:51.507330] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E43-0-0] [lt=60] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:51.589759] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D52-0-0] [lt=22] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:51.591174] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D52-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:51.591188] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D52-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:51.679631] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E3F-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:51.811746] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E42-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:52.007065] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CCD-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:52.009300] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CCD-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:52.266030] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D3B-0-0] [lt=15] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363111820886004, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363111820886005, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363111820886005, v:0}}]) [2024-06-26 08:51:52.437264] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363110819405004, v:0}, replayable_scn:{val:1719363110819405004, v:0}, standby_scn:{val:1719363110819405004, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363112434339, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363110819405004, v:0}, replayable_scn:{val:1719363110819405004, v:0}, standby_scn:{val:1719363110819405004, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363112434339, ora_rowscn_=1719363112267387000) [2024-06-26 08:51:52.437769] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=85] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:52.437787] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:52.437792] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:52.507446] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E44-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:52.592861] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D53-0-0] [lt=22] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:52.594208] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D53-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:52.594222] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D53-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:52.679714] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E40-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:52.688082] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9F-0-0] [lt=7] start do ObArchiveSchedulerService round(round=29277) [2024-06-26 08:51:52.689576] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9F-0-0] [lt=12][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:51:52.689589] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9F-0-0] [lt=12][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:51:52.689611] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9F-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:52.689622] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9F-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:51:52.689627] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410B9F-0-0] [lt=5][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:52.811862] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E43-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:53.012420] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CCE-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:53.014064] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CCE-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:53.272161] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D3C-0-0] [lt=15] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363112822717003, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363112822717004, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363112822717004, v:0}}]) [2024-06-26 08:51:53.436060] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:53.436075] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:53.436079] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:53.442283] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:390) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B025-0-0] [lt=51] get number(ret=0, service_type_=0, range=1, base_id=1719363113441531000, start_id=1719363113441531000, end_id=1719363113441531001) [2024-06-26 08:51:53.443835] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:340) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B025-0-0] [lt=43] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1719363113441531000, v:0}, tenant_id=1002) [2024-06-26 08:51:53.507556] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E45-0-0] [lt=45] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:53.595937] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D54-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:53.597321] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D54-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:53.597337] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D54-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:53.674622] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=5] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363113434079, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363111820886005, v:0}, replayable_scn:{val:1719363111820886005, v:0}, standby_scn:{val:1719363111820886005, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:51:53.679780] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E41-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:53.811963] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E44-0-0] [lt=39] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:54.017315] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CCF-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:54.019282] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CCF-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:54.278482] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D3D-0-0] [lt=14] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363113823714003, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363113823714004, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363113823714004, v:0}}]) [2024-06-26 08:51:54.436125] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:54.436144] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:54.436149] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:54.436157] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=373, return_code_array=[0, 0, 0], broadcast_times=2) [2024-06-26 08:51:54.507672] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E46-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:54.599347] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D55-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:54.601012] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D55-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:54.601028] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D55-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:54.679867] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E42-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:54.689726] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA0-0-0] [lt=7] start do ObArchiveSchedulerService round(round=29278) [2024-06-26 08:51:54.691212] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA0-0-0] [lt=13][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:51:54.691224] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA0-0-0] [lt=11][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:51:54.691247] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA0-0-0] [lt=6][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:54.691259] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA0-0-0] [lt=12][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:51:54.691265] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA0-0-0] [lt=5][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:54.812040] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E45-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:55.022576] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CD0-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:55.024229] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CD0-0-0] [lt=26] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:55.284018] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D3E-0-0] [lt=15] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363114824654005, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363114824654005, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363114824654005, v:0}}]) [2024-06-26 08:51:55.288729] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:440) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D3E-0-0] [lt=18] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1719363112822717004, v:0}) [2024-06-26 08:51:55.436330] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:55.436344] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:55.436347] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:55.507781] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E47-0-0] [lt=42] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:55.602654] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D56-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:55.604012] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D56-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:55.604027] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D56-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:55.679952] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E43-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:55.812123] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E46-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:56.027493] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CD1-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:56.029309] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CD1-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:56.289973] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D3F-0-0] [lt=11] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363115824867004, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363115824867004, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363115824867004, v:0}}]) [2024-06-26 08:51:56.435959] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363114824654005, v:0}, replayable_scn:{val:1719363114824654005, v:0}, standby_scn:{val:1719363114824654005, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363116435245, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363114824654005, v:0}, replayable_scn:{val:1719363114824654005, v:0}, standby_scn:{val:1719363114824654005, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363116435245, ora_rowscn_=1719363116291091000) [2024-06-26 08:51:56.436292] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=28] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:56.436301] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:56.436304] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:56.507863] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E48-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:56.605758] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D57-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:56.607154] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D57-0-0] [lt=67] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:56.607167] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D57-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:56.676480] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=5] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363116435245, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363114824654005, v:0}, replayable_scn:{val:1719363114824654005, v:0}, standby_scn:{val:1719363114824654005, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:51:56.680035] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E44-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:56.691336] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA1-0-0] [lt=7] start do ObArchiveSchedulerService round(round=29279) [2024-06-26 08:51:56.692733] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA1-0-0] [lt=8][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:51:56.692748] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA1-0-0] [lt=14][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:51:56.692771] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA1-0-0] [lt=6][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:56.692782] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA1-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:51:56.692789] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA1-0-0] [lt=6][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:56.812202] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E47-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:57.032753] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CD2-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:57.034525] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CD2-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:57.295961] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D40-0-0] [lt=20] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363117267886000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363117267886000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363117267886000, v:0}}]) [2024-06-26 08:51:57.436292] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:57.436310] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:57.436314] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:57.453623] INFO [STORAGE.TRANS] alloc (ob_trans_factory.cpp:141) [62463][T1001_FrzInfoDe][T1001][YB420A0A0A34-00061BB2CCF0B026-0-0] [lt=14] ObTransCtx statistics(active_coord_ctx_count=0, active_part_ctx_count=1, total_release_part_ctx_count=110) [2024-06-26 08:51:57.453887] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:390) [62463][T1001_FrzInfoDe][T1001][YB420A0A0A34-00061BB2CCF0B026-0-0] [lt=41] get number(ret=0, service_type_=0, range=1, base_id=1719363117453711000, start_id=1719363117453711000, end_id=1719363117453711001) [2024-06-26 08:51:57.455647] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:340) [62463][T1001_FrzInfoDe][T1001][YB420A0A0A34-00061BB2CCF0B026-0-0] [lt=35] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1719363117453711000, v:0}, tenant_id=1001) [2024-06-26 08:51:57.507951] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E49-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:57.609112] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D58-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:57.610699] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D58-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:57.610717] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D58-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:57.680117] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E45-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:57.812285] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E48-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:58.037544] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CD3-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:58.039180] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CD3-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:58.302530] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D41-0-0] [lt=16] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363117827472004, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363117827472004, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363117827472004, v:0}}]) [2024-06-26 08:51:58.436849] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:58.436866] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:58.436870] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:58.436877] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=805, return_code_array=[0, 0, 0], broadcast_times=1) [2024-06-26 08:51:58.508032] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E4A-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:58.612352] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D59-0-0] [lt=22] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:58.613672] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D59-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:58.613685] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D59-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:58.680235] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E46-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:58.692862] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA2-0-0] [lt=6] start do ObArchiveSchedulerService round(round=29280) [2024-06-26 08:51:58.694552] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA2-0-0] [lt=16][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:51:58.694568] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA2-0-0] [lt=15][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:51:58.694594] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA2-0-0] [lt=6][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:58.694603] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA2-0-0] [lt=9][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:51:58.694609] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA2-0-0] [lt=5][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:51:58.812341] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E49-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:59.042550] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CD4-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:51:59.044207] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CD4-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:51:59.312561] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D42-0-0] [lt=45] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363118828325004, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363118828325003, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363118828325004, v:0}}]) [2024-06-26 08:51:59.314894] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:336) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D42-0-0] [lt=11] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1002, ls_id:{id:1}, sync_scn:{val:1719363118828325004, v:0}, readable_scn:{val:1719363118828325004, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:5, config_seq:6}}) [2024-06-26 08:51:59.436230] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363117827472004, v:0}, replayable_scn:{val:1719363117827472004, v:0}, standby_scn:{val:1719363117827472004, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363119435250, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363117827472004, v:0}, replayable_scn:{val:1719363117827472004, v:0}, standby_scn:{val:1719363117827472004, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363119435250, ora_rowscn_=1719363119313721000) [2024-06-26 08:51:59.436822] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=30] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:51:59.436835] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:51:59.436839] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:51:59.508111] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E4B-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:59.615294] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D5A-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:59.616648] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D5A-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:51:59.616663] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D5A-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:51:59.678434] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=4] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363119435250, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363117827472004, v:0}, replayable_scn:{val:1719363117827472004, v:0}, standby_scn:{val:1719363117827472004, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:51:59.680319] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E47-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:51:59.812424] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E4A-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:00.049109] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CD5-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:52:00.052153] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CD5-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:52:00.318598] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D43-0-0] [lt=15] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363119828782004, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363119828782004, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363119828782004, v:0}}]) [2024-06-26 08:52:00.436499] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:52:00.436518] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:52:00.436522] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:52:00.508192] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E4C-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:00.618361] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D5B-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:00.619691] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D5B-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:52:00.619705] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D5B-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:00.680401] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E48-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:00.694691] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA3-0-0] [lt=5] start do ObArchiveSchedulerService round(round=29281) [2024-06-26 08:52:00.700179] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA3-0-0] [lt=23][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:52:00.700217] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA3-0-0] [lt=36][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:52:00.700303] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA3-0-0] [lt=25][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:52:00.700338] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA3-0-0] [lt=34][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:52:00.700363] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA3-0-0] [lt=24][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:52:00.812504] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E4B-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:01.055519] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CD6-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:52:01.057284] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CD6-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:52:01.324722] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D44-0-0] [lt=17] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363120830212004, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363120830212004, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363120830212004, v:0}}]) [2024-06-26 08:52:01.436549] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:52:01.436564] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:52:01.436568] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:52:01.508272] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E4D-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:01.621404] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D5C-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:01.622757] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D5C-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:52:01.622772] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D5C-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:01.680527] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E49-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:01.812583] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E4C-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:02.061127] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CD7-0-0] [lt=23] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:52:02.063296] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CD7-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:52:02.330820] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D45-0-0] [lt=15] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363122269731000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363122328351000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363122328351000, v:0}}]) [2024-06-26 08:52:02.437853] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363120830212003, v:0}, replayable_scn:{val:1719363120830212003, v:0}, standby_scn:{val:1719363120830212003, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363122434962, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363120830212003, v:0}, replayable_scn:{val:1719363120830212003, v:0}, standby_scn:{val:1719363120830212003, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363122434962, ora_rowscn_=1719363122331540000) [2024-06-26 08:52:02.438288] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=89] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:52:02.438319] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=30] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:52:02.438332] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:52:02.438356] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=18] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=412, return_code_array=[0, 0, 0], broadcast_times=1) [2024-06-26 08:52:02.508349] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E4E-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:02.624451] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D5D-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:02.625827] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D5D-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:52:02.625839] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D5D-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:02.680385] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=5] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363122434962, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363120830212003, v:0}, replayable_scn:{val:1719363120830212003, v:0}, standby_scn:{val:1719363120830212003, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:52:02.680636] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E4A-0-0] [lt=45] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:02.700454] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA4-0-0] [lt=22] start do ObArchiveSchedulerService round(round=29282) [2024-06-26 08:52:02.701916] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA4-0-0] [lt=13][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:52:02.701928] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA4-0-0] [lt=11][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:52:02.701951] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA4-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:52:02.701962] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA4-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:52:02.701967] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA4-0-0] [lt=4][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:52:02.812626] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E4D-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:03.066639] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CD8-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:52:03.068590] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CD8-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:52:03.337071] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D46-0-0] [lt=18] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363122832858004, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363123333868001, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363123333868001, v:0}}]) [2024-06-26 08:52:03.436667] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=27] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:52:03.436682] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:52:03.436686] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:52:03.450744] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:390) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B026-0-0] [lt=39] get number(ret=0, service_type_=0, range=1, base_id=1719363123449854000, start_id=1719363123449854000, end_id=1719363123449854001) [2024-06-26 08:52:03.451583] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:340) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B026-0-0] [lt=18] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1719363123449854000, v:0}, tenant_id=1002) [2024-06-26 08:52:03.508429] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E4F-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:03.627841] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D5E-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:03.629479] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D5E-0-0] [lt=69] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:52:03.629494] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D5E-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:03.680721] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E4B-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:03.812706] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E4E-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:04.072489] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CD9-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:52:04.074388] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CD9-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:52:04.343397] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D47-0-0] [lt=20] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363123834576004, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363123834576004, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363123834576004, v:0}}]) [2024-06-26 08:52:04.438619] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:52:04.438668] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=47] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:52:04.438683] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:52:04.508538] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E50-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:04.631156] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D5F-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:04.632473] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D5F-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:52:04.632487] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D5F-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:04.680804] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E4C-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:04.702044] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA5-0-0] [lt=6] start do ObArchiveSchedulerService round(round=29283) [2024-06-26 08:52:04.703513] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA5-0-0] [lt=22][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:52:04.703528] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA5-0-0] [lt=14][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:52:04.703551] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA5-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:52:04.703559] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA5-0-0] [lt=7][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:52:04.703563] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA5-0-0] [lt=4][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:52:04.812782] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E4F-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:05.078519] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CDA-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:52:05.080593] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CDA-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:52:05.349437] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D48-0-0] [lt=13] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363124835592005, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363124835592005, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363124835592005, v:0}}]) [2024-06-26 08:52:05.353793] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:440) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D48-0-0] [lt=15] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1719363122832858004, v:0}) [2024-06-26 08:52:05.436979] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:52:05.436994] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:52:05.436999] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:52:05.508651] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E51-0-0] [lt=44] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:05.634194] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D60-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:05.635573] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D60-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:52:05.635586] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D60-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:05.680900] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E4D-0-0] [lt=22] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:05.682337] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363125435047, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363123834576004, v:0}, replayable_scn:{val:1719363123834576004, v:0}, standby_scn:{val:1719363123834576004, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:52:05.812839] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E50-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:06.084133] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CDB-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:52:06.085997] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CDB-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:52:06.355221] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D49-0-0] [lt=20] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363125837001004, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363125837001004, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363125837001004, v:0}}]) [2024-06-26 08:52:06.436604] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363124835592005, v:0}, replayable_scn:{val:1719363124835592005, v:0}, standby_scn:{val:1719363124835592005, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363126435047, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363124835592005, v:0}, replayable_scn:{val:1719363124835592005, v:0}, standby_scn:{val:1719363124835592005, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363126435047, ora_rowscn_=1719363126356239000) [2024-06-26 08:52:06.436965] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=26] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:52:06.436977] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:52:06.436980] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:52:06.436985] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=355, return_code_array=[0, 0, 0], broadcast_times=2) [2024-06-26 08:52:06.445102] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:126) [61869][T1001_REST_SER][T1001][YB420A0A0A34-00061BB2CD00A884-0-0] [lt=10] [RESTORE] restore scheduler quit [2024-06-26 08:52:06.445643] INFO [SHARE] get_all_recover_table_job (ob_recover_table_persist_helper.cpp:199) [61869][T1001_REST_SER][T1001][YB420A0A0A34-00061BB2CD00A884-0-0] [lt=13] get targets(ret=0, jobs=[], exec_tenant_id=1001, sql=select * from __all_recover_table_job) [2024-06-26 08:52:06.446202] INFO [SHARE] get_all_import_table_jobs (ob_import_table_persist_helper.cpp:109) [61869][T1001_REST_SER][T1001][YB420A0A0A34-00061BB2CD00A885-0-0] [lt=14] get targets(ret=0, jobs=[], exec_tenant_id=1001, sql=select * from __all_import_table_job) [2024-06-26 08:52:06.446222] WDIAG [SHARE.SCHEMA] revert (ob_schema_mgr_cache.cpp:122) [61869][T1001_REST_SER][T1001][YB420A0A0A34-00061BB2CD00A886-0-0] [lt=15][errcode=0] long time to hold one guard(schema_mgr=0x2abf03d15260, tenant_id=1, version=1719298380350128, cur_timestamp=1719363126446218, ref_timestamp=1719363066444565, lbt()="0x17a30451 0x1523bc9f 0x5f9b859 0x5f9bdc0 0x5ff915f 0xac59643 0x8f5ffef 0x62d7da4 0x17b107dc 0x17b1be21 0x17b1814e 0x2abe35ce2dc5 0x2abe35fee73d") [2024-06-26 08:52:06.446301] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:95) [61869][T1001_REST_SER][T1001][YB420A0A0A34-00061BB2CD00A887-0-0] [lt=7] [RESTORE] restore scheduler start [2024-06-26 08:52:06.446308] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:102) [61869][T1001_REST_SER][T1001][YB420A0A0A34-00061BB2CD00A888-0-0] [lt=7] [RESTORE] try process restore job [2024-06-26 08:52:06.446732] INFO [COMMON] get_jobs (ob_physical_restore_table_operator.cpp:416) [61869][T1001_REST_SER][T1001][YB420A0A0A34-00061BB2CD00A888-0-0] [lt=2] [RESTORE] get restore jobs(ret=0, job_cnt=0) [2024-06-26 08:52:06.508732] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E52-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:06.637297] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D61-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:06.638679] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D61-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:52:06.638695] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D61-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:06.680983] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E4E-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:06.703634] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA6-0-0] [lt=4] start do ObArchiveSchedulerService round(round=29284) [2024-06-26 08:52:06.705094] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA6-0-0] [lt=13][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:52:06.705108] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA6-0-0] [lt=13][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:52:06.705130] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA6-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:52:06.705141] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA6-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:52:06.705146] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA6-0-0] [lt=4][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:52:06.812919] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E51-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:07.088918] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CDC-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:52:07.090876] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CDC-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:52:07.361999] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D4A-0-0] [lt=14] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363127272461000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363127330016000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363127330016000, v:0}}]) [2024-06-26 08:52:07.438696] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:52:07.438740] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=43] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:52:07.438755] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:52:07.460580] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:390) [62463][T1001_FrzInfoDe][T1001][YB420A0A0A34-00061BB2CCF0B027-0-0] [lt=38] get number(ret=0, service_type_=0, range=1, base_id=1719363127460193000, start_id=1719363127460193000, end_id=1719363127460193001) [2024-06-26 08:52:07.461366] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:340) [62463][T1001_FrzInfoDe][T1001][YB420A0A0A34-00061BB2CCF0B027-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1719363127460193000, v:0}, tenant_id=1001) [2024-06-26 08:52:07.508811] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E53-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:07.640508] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D62-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:07.642197] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D62-0-0] [lt=50] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:52:07.642210] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D62-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:07.681064] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E4F-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:07.812994] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E52-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:08.093876] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CDD-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:52:08.095448] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CDD-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:52:08.371362] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D4B-0-0] [lt=41] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363127839525004, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363128340312000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363127839525004, v:0}}]) [2024-06-26 08:52:08.437337] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:52:08.437355] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:52:08.437359] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:52:08.508922] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E54-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:08.643880] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D63-0-0] [lt=22] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:08.645260] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D63-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:52:08.645275] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D63-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:08.681141] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E50-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:08.684255] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=5] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363128435945, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363127338496000, v:0}, replayable_scn:{val:1719363127338496000, v:0}, standby_scn:{val:1719363127338496000, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:52:08.705222] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA7-0-0] [lt=6] start do ObArchiveSchedulerService round(round=29285) [2024-06-26 08:52:08.706660] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA7-0-0] [lt=14][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:52:08.706682] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA7-0-0] [lt=22][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:52:08.706706] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA7-0-0] [lt=6][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:52:08.706717] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA7-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:52:08.706723] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA7-0-0] [lt=5][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:52:08.813070] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E53-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:09.098402] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CDE-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:52:09.100029] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CDE-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:52:09.377082] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D4C-0-0] [lt=15] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363128840256004, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363129340952000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363129340952000, v:0}}]) [2024-06-26 08:52:09.379432] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:336) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D4C-0-0] [lt=14] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1002, ls_id:{id:1}, sync_scn:{val:1719363129340952000, v:0}, readable_scn:{val:1719363129340952000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:5, config_seq:6}}) [2024-06-26 08:52:09.436844] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363128340312000, v:0}, replayable_scn:{val:1719363128340312000, v:0}, standby_scn:{val:1719363128340312000, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363129435715, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363128340312000, v:0}, replayable_scn:{val:1719363128340312000, v:0}, standby_scn:{val:1719363128340312000, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363129435715, ora_rowscn_=1719363129378147000) [2024-06-26 08:52:09.437164] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=28] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:52:09.437174] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:52:09.437177] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:52:09.437181] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=309, return_code_array=[0, 0, 0], broadcast_times=2) [2024-06-26 08:52:09.509029] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E55-0-0] [lt=38] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:09.646909] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D64-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:09.648294] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D64-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:52:09.648309] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D64-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:09.681220] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E51-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:09.813152] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E54-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:10.103139] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CDF-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:52:10.104891] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CDF-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:52:10.383671] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D4D-0-0] [lt=20] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363129841192004, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363130342181000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363130342181000, v:0}}]) [2024-06-26 08:52:10.437293] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:52:10.437310] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:52:10.437314] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:52:10.509119] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E56-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:10.651815] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D65-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:10.657424] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D65-0-0] [lt=186] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:52:10.657494] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D65-0-0] [lt=69] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:10.681303] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E52-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:10.706801] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA8-0-0] [lt=4] start do ObArchiveSchedulerService round(round=29286) [2024-06-26 08:52:10.708224] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA8-0-0] [lt=12][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:52:10.708238] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA8-0-0] [lt=13][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:52:10.708261] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA8-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:52:10.708271] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA8-0-0] [lt=10][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:52:10.708276] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA8-0-0] [lt=5][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:52:10.813233] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E55-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:11.108106] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CE0-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:52:11.109812] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CE0-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:52:11.395474] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D4E-0-0] [lt=52] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363131343047000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363131343047000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363131343047000, v:0}}]) [2024-06-26 08:52:11.437322] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:52:11.437339] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:52:11.437343] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:52:11.509202] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E57-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:11.659269] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D66-0-0] [lt=80] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:11.660592] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D66-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:52:11.660603] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D66-0-0] [lt=11] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:11.681402] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E53-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:11.686178] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=5] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363131435528, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363130342181000, v:0}, replayable_scn:{val:1719363130342181000, v:0}, standby_scn:{val:1719363130342181000, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:52:11.813316] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E56-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:12.052446] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:714) [61872][T1001_BackupDat][T1001][YB420A0A0A34-00061BB2CD10A1AA-0-0] [lt=14] [DATA_BACKUP]success get jobs(job_attrs=[]) [2024-06-26 08:52:12.097971] INFO [RS] dump_statistics (ob_backup_task_scheduler.cpp:245) [61871][T1001_BACKUP_SC][T1001][YB420A0A0A34-00061BB2CD209943-0-0] [lt=13] backup scheduler task statistics(waiting_task_cnt=0, executing_task_cnt=0) [2024-06-26 08:52:12.113034] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CE1-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:52:12.113568] INFO [RS] try_update_primary_ip_list (ob_common_ls_service.cpp:360) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CE1-0-0] [lt=13] log restore source is empty, just skip(ret=-4018, user_tenant_id=1002) [2024-06-26 08:52:12.115373] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CE1-0-0] [lt=5] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:52:12.401365] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D4F-0-0] [lt=16] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363132332192000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363132332192000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363132332192000, v:0}}]) [2024-06-26 08:52:12.437076] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363131343047000, v:0}, replayable_scn:{val:1719363131343047000, v:0}, standby_scn:{val:1719363131343047000, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363132436140, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363131343047000, v:0}, replayable_scn:{val:1719363131343047000, v:0}, standby_scn:{val:1719363131343047000, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363132436140, ora_rowscn_=1719363132402133000) [2024-06-26 08:52:12.437406] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=27] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:52:12.437420] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:52:12.437423] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:52:12.437429] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=325, return_code_array=[0, 0, 0], broadcast_times=1) [2024-06-26 08:52:12.509278] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E58-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:12.662310] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D67-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:12.663657] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D67-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:52:12.663670] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D67-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:12.681497] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E54-0-0] [lt=30] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:12.708352] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA9-0-0] [lt=7] start do ObArchiveSchedulerService round(round=29287) [2024-06-26 08:52:12.709838] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA9-0-0] [lt=12][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:52:12.709852] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA9-0-0] [lt=13][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:52:12.709875] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA9-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:52:12.709886] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA9-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:52:12.709891] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BA9-0-0] [lt=5][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:52:12.813395] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E57-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:13.118467] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CE2-0-0] [lt=16] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:52:13.121218] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CE2-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:52:13.407570] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D50-0-0] [lt=17] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363133346131001, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363133346131001, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363133346131001, v:0}}]) [2024-06-26 08:52:13.437545] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:52:13.437562] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:52:13.437567] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:52:13.455102] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:390) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B027-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1719363133454888000, start_id=1719363133454888000, end_id=1719363133454888001) [2024-06-26 08:52:13.455975] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:340) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B027-0-0] [lt=13] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1719363133454888000, v:0}, tenant_id=1002) [2024-06-26 08:52:13.509359] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E59-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:13.669894] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D68-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:13.675510] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D68-0-0] [lt=206] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:52:13.675553] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D68-0-0] [lt=42] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:13.681609] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E55-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:13.813473] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E58-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:14.124666] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CE3-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:52:14.126550] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CE3-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:52:14.415095] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D51-0-0] [lt=16] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363134346707000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363134346707000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363134346707000, v:0}}]) [2024-06-26 08:52:14.437408] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:52:14.437427] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:52:14.437431] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:52:14.509479] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E5A-0-0] [lt=39] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:14.677350] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D69-0-0] [lt=80] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:14.678727] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D69-0-0] [lt=50] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:52:14.678740] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D69-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:14.681718] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E56-0-0] [lt=39] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:14.688085] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=10] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363134436302, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363133346131000, v:0}, replayable_scn:{val:1719363133346131000, v:0}, standby_scn:{val:1719363133346131000, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:52:14.709956] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BAA-0-0] [lt=4] start do ObArchiveSchedulerService round(round=29288) [2024-06-26 08:52:14.711452] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BAA-0-0] [lt=12][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:52:14.711465] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BAA-0-0] [lt=13][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:52:14.711489] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BAA-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:52:14.711499] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BAA-0-0] [lt=10][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:52:14.711504] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BAA-0-0] [lt=5][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:52:14.813546] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E59-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:15.129797] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CE4-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:52:15.131507] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CE4-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:52:15.420938] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D52-0-0] [lt=16] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363135267213000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363135267213000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363135267213000, v:0}}]) [2024-06-26 08:52:15.425766] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:440) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D52-0-0] [lt=14] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1719363133346131000, v:0}) [2024-06-26 08:52:15.437300] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363134318018001, v:0}, replayable_scn:{val:1719363134318018001, v:0}, standby_scn:{val:1719363134318018001, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363135436372, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363134318018001, v:0}, replayable_scn:{val:1719363134318018001, v:0}, standby_scn:{val:1719363134318018001, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363135436372, ora_rowscn_=1719363135422512000) [2024-06-26 08:52:15.437553] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=26] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:52:15.437566] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:52:15.437569] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:52:15.437574] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=247, return_code_array=[0, 0, 0], broadcast_times=2) [2024-06-26 08:52:15.509553] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E5B-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:15.680490] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D6A-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:15.681814] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E57-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:15.681879] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D6A-0-0] [lt=52] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:52:15.681892] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D6A-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:15.813625] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E5A-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:16.134628] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CE5-0-0] [lt=16] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:52:16.136235] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CE5-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:52:16.427069] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D53-0-0] [lt=12] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363136368100000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363136368100000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363136368100000, v:0}}]) [2024-06-26 08:52:16.437520] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:52:16.437532] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:52:16.437538] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:52:16.509651] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E5C-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:16.681900] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E58-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:16.683890] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D6B-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:16.685662] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D6B-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:52:16.685676] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D6B-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:16.711577] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BAB-0-0] [lt=7] start do ObArchiveSchedulerService round(round=29289) [2024-06-26 08:52:16.713049] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BAB-0-0] [lt=14][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:52:16.713074] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BAB-0-0] [lt=24][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:52:16.713097] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BAB-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:52:16.713108] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BAB-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:52:16.713113] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BAB-0-0] [lt=4][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:52:16.813735] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E5B-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:17.139437] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CE6-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:52:17.141221] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CE6-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:52:17.432675] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D54-0-0] [lt=15] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363137333759000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363137333759000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363137333759000, v:0}}]) [2024-06-26 08:52:17.437638] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:52:17.437654] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:52:17.437658] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:52:17.465102] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:390) [62463][T1001_FrzInfoDe][T1001][YB420A0A0A34-00061BB2CCF0B028-0-0] [lt=15] get number(ret=0, service_type_=0, range=1, base_id=1719363137464851000, start_id=1719363137464851000, end_id=1719363137464851001) [2024-06-26 08:52:17.465872] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:340) [62463][T1001_FrzInfoDe][T1001][YB420A0A0A34-00061BB2CCF0B028-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1719363137464851000, v:0}, tenant_id=1001) [2024-06-26 08:52:17.509732] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E5D-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:17.681984] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E59-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:17.687421] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D6C-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:17.688762] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D6C-0-0] [lt=51] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:52:17.688774] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D6C-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:17.690058] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=5] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363137436166, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363136425538000, v:0}, replayable_scn:{val:1719363136425538000, v:0}, standby_scn:{val:1719363136425538000, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:52:17.813842] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E5C-0-0] [lt=44] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:18.144420] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CE7-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:52:18.146117] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CE7-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:52:18.437436] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363136425538000, v:0}, replayable_scn:{val:1719363136425538000, v:0}, standby_scn:{val:1719363136425538000, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363138436287, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363136425538000, v:0}, replayable_scn:{val:1719363136425538000, v:0}, standby_scn:{val:1719363136425538000, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363138436287, ora_rowscn_=1719363137432976000) [2024-06-26 08:52:18.437887] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D55-0-0] [lt=16] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363138351074000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363138351074000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363138351074000, v:0}}]) [2024-06-26 08:52:18.509815] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E5E-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:18.682067] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E5A-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:18.690453] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D6D-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:18.691826] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D6D-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:52:18.691840] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D6D-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:18.713188] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BAC-0-0] [lt=4] start do ObArchiveSchedulerService round(round=29290) [2024-06-26 08:52:18.714636] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BAC-0-0] [lt=15][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:52:18.714652] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BAC-0-0] [lt=15][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:52:18.714675] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BAC-0-0] [lt=5][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:52:18.714685] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BAC-0-0] [lt=10][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:52:18.714690] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BAC-0-0] [lt=5][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:52:18.813933] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E5D-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:19.149200] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CE8-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:52:19.150829] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CE8-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:52:19.437767] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=24] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:52:19.437780] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:52:19.437784] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:52:19.437788] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=255, return_code_array=[0, 0, 0], broadcast_times=1) [2024-06-26 08:52:19.443448] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D56-0-0] [lt=14] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363139351538000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363139351538000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363139351538000, v:0}}]) [2024-06-26 08:52:19.445646] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:336) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D56-0-0] [lt=16] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1002, ls_id:{id:1}, sync_scn:{val:1719363139351538000, v:0}, readable_scn:{val:1719363139351538000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:5, config_seq:6}}) [2024-06-26 08:52:19.509892] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E5F-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:19.682143] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E5B-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:19.693488] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D6E-0-0] [lt=22] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:19.694856] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D6E-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:52:19.694867] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D6E-0-0] [lt=10] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:19.814022] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E5E-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:20.160177] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CE9-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:52:20.162998] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CE9-0-0] [lt=54] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:52:20.437818] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:52:20.437837] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:52:20.437840] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:52:20.449180] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D57-0-0] [lt=17] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363140352712000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363140352712000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363140352712000, v:0}}]) [2024-06-26 08:52:20.509983] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E60-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:20.682223] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E5C-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:20.691937] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=5] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363140436663, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363138351074000, v:0}, replayable_scn:{val:1719363138351074000, v:0}, standby_scn:{val:1719363138351074000, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:52:20.696513] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D6F-0-0] [lt=43] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:20.697845] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D6F-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:52:20.697859] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D6F-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:20.714791] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BAD-0-0] [lt=7] start do ObArchiveSchedulerService round(round=29291) [2024-06-26 08:52:20.716525] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BAD-0-0] [lt=17][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:52:20.716538] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BAD-0-0] [lt=12][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:52:20.716565] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BAD-0-0] [lt=6][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:52:20.716576] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BAD-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:52:20.716584] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BAD-0-0] [lt=8][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:52:20.814108] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E5F-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:21.166575] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CEA-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:52:21.168481] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CEA-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:52:21.437611] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:627) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363139351538000, v:0}, replayable_scn:{val:1719363139351538000, v:0}, standby_scn:{val:1719363139351538000, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, new_refresh_time_us=1719363141436225, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363139351538000, v:0}, replayable_scn:{val:1719363139351538000, v:0}, standby_scn:{val:1719363139351538000, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}, last_sql_update_time_=1719363141436225, ora_rowscn_=1719363140450421000) [2024-06-26 08:52:21.437878] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=25] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:52:21.437890] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:52:21.437893] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:52:21.455129] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D58-0-0] [lt=18] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363141353480000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363141353480000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363141353480000, v:0}}]) [2024-06-26 08:52:21.510029] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E61-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:21.682306] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E5D-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:21.699508] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D70-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:21.700818] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D70-0-0] [lt=51] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:52:21.700832] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D70-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:21.814190] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E60-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:22.171729] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CEB-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:52:22.173876] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CEB-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:52:22.437907] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:52:22.437920] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:52:22.437924] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:52:22.437928] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:376) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=244, return_code_array=[0, 0, 0], broadcast_times=1) [2024-06-26 08:52:22.460713] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D59-0-0] [lt=15] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363142335412000, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363142335412000, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363142335412000, v:0}}]) [2024-06-26 08:52:22.510080] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E62-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:22.682388] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E5E-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:22.702494] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D71-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:22.703820] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D71-0-0] [lt=52] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:52:22.703834] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D71-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:22.716661] INFO [RS] run2 (ob_archive_scheduler_service.cpp:84) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BAE-0-0] [lt=9] start do ObArchiveSchedulerService round(round=29292) [2024-06-26 08:52:22.718096] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:947) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BAE-0-0] [lt=13][errcode=-4018] no row exist(ret=-4018) [2024-06-26 08:52:22.718110] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BAE-0-0] [lt=13][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-06-26 08:52:22.718132] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BAE-0-0] [lt=4][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:52:22.718142] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BAE-0-0] [lt=10][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-06-26 08:52:22.718147] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [61874][T1001_ArchiveSv][T1001][YB420A0A0A34-00061BB2CD410BAE-0-0] [lt=5][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-06-26 08:52:22.814272] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E61-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:23.178045] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CEC-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:52:23.179913] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CEC-0-0] [lt=27] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-06-26 08:52:23.438012] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.10.10.51:2882") [2024-06-26 08:52:23.438027] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.10.10.52:2882") [2024-06-26 08:52:23.438031] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:367) [62039][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.10.10.53:2882") [2024-06-26 08:52:23.459644] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:390) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B028-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1719363143459567000, start_id=1719363143459567000, end_id=1719363143459567001) [2024-06-26 08:52:23.460671] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:340) [62478][T1002_FrzInfoDe][T1002][YB420A0A0A34-00061BB2CE10B028-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1719363143459567000, v:0}, tenant_id=1002) [2024-06-26 08:52:23.466482] INFO [RS] gather_replica_readable_scn (ob_ls_recovery_stat_handler.cpp:494) [62042][T1002_TeRec][T1002][YB420A0A0A34-00061BB2C7817D5A-0-0] [lt=9] ls readable scn in memory(ret=0, ret="OB_SUCCESS", ls_id={id:1}, replicas_scn_=[{server:"10.10.10.52:2882", readable_scn:{val:1719363143355481001, v:0}}, {server:"10.10.10.51:2882", readable_scn:{val:1719363143459567001, v:0}}, {server:"10.10.10.53:2882", readable_scn:{val:1719363143459567001, v:0}}]) [2024-06-26 08:52:23.510143] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61632][T1_StandbySchem][T1][YB420A0A0A34-00061BB2C5317E63-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:23.682474] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [62041][T1002_StandbySc][T1002][YB420A0A0A34-00061BB2C7717E5F-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:23.693726] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:414) [62149][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=6] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1719363143436153, tenant_info={tenant_id:1002, tenant_role:{value:1}, switchover_status:{value:1}, switchover_epoch:0, sync_scn:{val:1719363141353480000, v:0}, replayable_scn:{val:1719363141353480000, v:0}, standby_scn:{val:1719363141353480000, v:0}, recovery_until_scn:{val:4611686018427387903, v:0}, log_mode:{archive mode:"NOARCHIVELOG"}, max_ls_id:{id:1003}}) [2024-06-26 08:52:23.705494] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D72-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007170059000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1002, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007175389000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1002, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1718183007177545001, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:23.706839] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D72-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-06-26 08:52:23.706852] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [62044][T1002_PLSSer][T1002][YB420A0A0A34-00061BB2CE217D72-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1718183007270864, tenant_name:"ocp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-06-26 08:52:23.814328] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [61863][T1001_StandbySc][T1001][YB420A0A0A34-00061BB2C5F17E62-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-06-26 08:52:24.183129] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CED-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-06-26 08:52:24.184837] INFO [RS] do_work (ob_common_ls_service.cpp:138) [61864][T1001_COMMONLSS][T1001][YB420A0A0A34-00061BB2CD517CED-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000)