[2024-12-02 13:55:05.774540] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF594-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:05.839351] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAC6-0-0] [lt=22] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:05.862018] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF24E-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:05.875454] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF24E-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:05.875465] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF24E-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:05.875480] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF24E-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:05.875489] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF24E-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:05.878784] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF595-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:05.940075] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC37-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:05.952073] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=11] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1020, epoch_=0, all_service_names_=[]) [2024-12-02 13:55:05.980397] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF24F-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:05.981959] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF24F-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:05.981970] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF24F-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:05.981985] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF24F-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:05.981994] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF24F-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:05.982230] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF596-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.026960] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECAD-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:55:06.030444] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAC2-0-0] [lt=14] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:06.032185] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECAD-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:55:06.051978] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC3F-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:06.085861] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF597-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.086893] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF250-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:06.088433] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF250-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.088443] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF250-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:06.088458] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF250-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:06.088467] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF250-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:06.110178] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC25-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:06.112729] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC25-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:06.112750] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC25-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:06.132725] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:06.134100] WDIAG [RS] get_dbms_job_infos_in_tenant (ob_dbms_job_utils.cpp:363) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=4][errcode=-5019] execute query failed(ret=-5019, sql=select * from __all_job, tenant_id=1020) [2024-12-02 13:55:06.134155] WDIAG [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:613) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=9][errcode=-5019] fail to exec job_utils_.get_dbms_job_infos_in_tenant(tenant_id, allocator, job_infos)(ret=-5019) [2024-12-02 13:55:06.134166] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10] load and register new jobs(ret=-5019, job_key=NULL, job_key=NULL, job_infos=[]) [2024-12-02 13:55:06.134177] WDIAG [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:590) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11][errcode=-5019] fail to exec load_and_register_new_jobs(tenant_ids.at(i))(ret=-5019) [2024-12-02 13:55:06.134185] INFO [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:593) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7] register single tenant(ret=-5019, tenant_ids.at(i)=1020, tenant_schema->get_compatibility_mode()=0) [2024-12-02 13:55:06.134197] WDIAG [RS] scheduler_job (ob_dbms_job_master.cpp:368) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=9][errcode=-5019] fail to exec load_and_register_all_jobs(job_key)(ret=-5019) [2024-12-02 13:55:06.134204] WDIAG [RS] scheduler (ob_dbms_job_master.cpp:340) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7][errcode=0] fail to scheduler single dbms job(ret=0, tmp_ret=-5019, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1733118911132741, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:55:06.189118] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF598-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.193349] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF251-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:06.194993] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEA94-0-0] [lt=21] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:06.195219] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF251-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.195230] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF251-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:06.195246] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF251-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:06.195255] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF251-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:06.209028] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:06.210185] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=18] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118906208962, delay:5000000, job_id:0, tenant_id:1019}, job_key=0xfffeb4af6120, job_infos=[]) [2024-12-02 13:55:06.210213] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=25] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118911210211, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:55:06.235763] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC43-0-0] [lt=35] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:06.293296] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF599-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.300145] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF252-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:06.301511] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF252-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.301521] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF252-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:06.301536] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF252-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:06.301545] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF252-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:06.303947] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:06.305099] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=18] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118906303883, delay:5000000, job_id:0, tenant_id:1001}, job_key=0xfffeb4af60a0, job_infos=[]) [2024-12-02 13:55:06.305126] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=24] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118911305123, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-02 13:55:06.306253] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118906304998, delay:5000000, job_id:0, tenant_id:1002}, job_key=0xfffeb4af60e0, job_infos=[]) [2024-12-02 13:55:06.306279] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=24] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118911306277, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-02 13:55:06.312083] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEA94-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:06.352006] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:06.352025] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=14] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=260, return_code_array=[0], broadcast_times=1) [2024-12-02 13:55:06.353311] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-02 13:55:06.397056] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF59A-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.406434] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF253-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:06.407968] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF253-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.407979] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF253-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:06.407995] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF253-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:06.408001] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF253-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:06.480586] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC46-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:06.500891] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF59B-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.512892] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF254-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:06.514099] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF254-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.514109] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF254-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:06.514125] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF254-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:06.514134] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF254-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:06.535175] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC26-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:06.536335] INFO [RS] try_update_primary_ip_list (ob_common_ls_service.cpp:361) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC26-0-0] [lt=18] log restore source is empty, just skip(ret=-4018, user_tenant_id=1002) [2024-12-02 13:55:06.538800] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC26-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:06.558879] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA7B-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:06.561274] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA7B-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:06.604384] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF59C-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.618940] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF255-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:06.620500] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF255-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.620510] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF255-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:06.620526] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF255-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:06.620535] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF255-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:06.708058] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF59D-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.725414] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF256-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:06.726611] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF256-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.726622] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF256-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:06.726637] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF256-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:06.726646] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF256-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:06.759042] INFO [RS.COMPACTION] can_start_work (ob_freeze_info_detector.cpp:265) [2325689][T1020_FrzInfoDe][T1020][YB42C0A800AF-00062843179EE965-0-0] [lt=31] tenant is in abnormal status, no need detect now(tenant_id=1020, tenant_schema={tenant_id:1020, schema_version:1733118578035224, tenant_name:"mysql_hy", name_case_mode:-1, read_only:false, primary_zone:"RANDOM", locality:"FULL{1}@zone1", previous_locality:"", compatibility_mode:0, gmt_modified:1733118578035236, drop_tenant_time:-1, status:3, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:06.811616] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF59E-0-0] [lt=21][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.831580] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF257-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:06.832862] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF257-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.832872] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF257-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:06.832886] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF257-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:06.832894] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF257-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:06.915288] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF59F-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.931205] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:464) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF59F-0-0] [lt=52] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.932560] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE965-0-0] [lt=28] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:06.933784] INFO [RS] do_ls_balance_task_ (ob_recovery_ls_service.cpp:1308) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE965-0-0] [lt=15] no need process balance task(tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}) [2024-12-02 13:55:06.934969] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE965-0-0] [lt=15] [LS_RECOVERY] finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, start_scn={val:18446744073709551615, v:3}, thread_idx=0, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:06.937764] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF258-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:06.938987] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF258-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.938994] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF258-0-0] [lt=5][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:06.939008] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF258-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:06.939016] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF258-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:06.941414] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC38-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:07.031530] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5A0-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.043703] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECAE-0-0] [lt=11] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118906703958001, v:0}}], last_dump_ts_=1733118901959829, force_dump=false) [2024-12-02 13:55:07.043730] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECAE-0-0] [lt=23] is primary tenant, no need check readable_scn [2024-12-02 13:55:07.043937] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF259-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:07.045168] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF259-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.045178] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF259-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:07.045193] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF259-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:07.045200] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF259-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:07.048844] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECAE-0-0] [lt=8] is primary tenant, no need check readable_scn [2024-12-02 13:55:07.052056] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC40-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:07.115438] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC26-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:07.117971] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC26-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:07.117989] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC26-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:07.135181] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5A1-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.150023] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25A-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:07.151661] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25A-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.151671] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25A-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:07.151686] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25A-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:07.151693] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25A-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:07.195069] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEA95-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:07.235859] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC44-0-0] [lt=33] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:07.238917] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5A2-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.256548] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25B-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:07.257865] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25B-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.257875] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25B-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:07.257890] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25B-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:07.257898] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:07.312158] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEA95-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:07.342276] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5A3-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.351868] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118905555272004, replayable_scn:1733118905555272004, readable_scn:1733118905555272003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118907348911, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118905555272004, replayable_scn:1733118905555272004, readable_scn:1733118905555272003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118907348911, ora_rowscn_=1733118907045839000, content_changed=true) [2024-12-02 13:55:07.352080] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=21] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:07.362769] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25C-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:07.364585] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25C-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.364594] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25C-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:07.364608] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25C-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:07.364617] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25C-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:07.403110] INFO [RS] check_lost_replicas (ob_lost_replica_checker.cpp:117) [2319631][LostRepCheck][T0][Y0-0000000000000000-0-0] [lt=14] start checking lost replicas [2024-12-02 13:55:07.404899] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=15] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118907348911, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118905555272004, replayable_scn:1733118905555272004, readable_scn:1733118905555272003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:55:07.445707] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5A4-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.469556] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25D-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:07.470773] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25D-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.470783] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25D-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:07.470797] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25D-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:07.470805] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25D-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:07.480660] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC47-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:07.544049] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC27-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:07.546464] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC27-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:07.549270] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5A5-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.552325] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:345) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5A5-0-0] [lt=52] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1020, ls_id:{id:1001}, sync_scn:{val:1732874426991810003, v:0}, readable_scn:{val:1732874426991810002, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:07.568030] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA7C-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:07.570477] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA7C-0-0] [lt=28] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:07.575623] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25E-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:07.577160] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25E-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.577172] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25E-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:07.577189] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25E-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:07.577199] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25E-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:07.620440] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319551][LeaseQueueTh2][T0][YB42C0A800AF-000628430B4EEAC1-0-0] [lt=21] [DDL] execute ddl like stmt(ret=0, cost=12, ddl_arg=NULL) [2024-12-02 13:55:07.652791] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5A6-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.682076] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25F-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:07.683334] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25F-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.683345] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25F-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:07.683361] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25F-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:07.683370] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25F-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:07.756570] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5A7-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.788180] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF260-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:07.789649] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF260-0-0] [lt=0] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:07.844117] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAC7-0-0] [lt=24] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:07.851796] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118907848637, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118907848637, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:55:07.860174] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5A8-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.894387] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF261-0-0] [lt=20] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:07.895811] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF261-0-0] [lt=0] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:07.920783] INFO [RS] finish (ob_mview_maintenance_task.cpp:282) [2319901][T1_TntSharedTim][T1][Y0-0000000000000000-0-0] [lt=19] mview maintenace task finish(tenant_id_=1, round_=10, status_=100, error_code_=0, cost_us_=20001259, prepare_cost_us_=0, gc_mview_cost_us_=1095, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2024-12-02 13:55:07.939416] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:452) [2319901][T1_TntSharedTim][T1][Y0-0000000000000000-0-0] [lt=13] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2024-12-02 13:55:07.942772] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC39-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:07.965056] INFO [RS] run2 (ob_archive_scheduler_service.cpp:85) [2319874][T1_ArchiveSvr][T1][YB42C0A800AF-0006284308FEE95C-0-0] [lt=10] start do ObArchiveSchedulerService round(round=17) [2024-12-02 13:55:07.967273] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:954) [2319874][T1_ArchiveSvr][T1][YB42C0A800AF-0006284308FEE95C-0-0] [lt=19][errcode=-4018] REACH SYSLOG RATE LIMIT [bandwidth] [2024-12-02 13:55:07.970062] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:717) [2319871][T1_BackupDataSr][T1][YB42C0A800AF-0006284308DEE966-0-0] [lt=18] [DATA_BACKUP]success get jobs(job_attrs=[]) [2024-12-02 13:55:07.976737] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5A9-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.000656] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF262-0-0] [lt=25] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:08.002128] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF262-0-0] [lt=0] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:08.030752] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAC3-0-0] [lt=11] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:08.052130] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC41-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:08.053536] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECAF-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:55:08.058587] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECAF-0-0] [lt=11] is primary tenant, no need check readable_scn [2024-12-02 13:55:08.080157] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5AA-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.106848] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF263-0-0] [lt=26] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:08.108661] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF263-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.108672] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF263-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:08.108688] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF263-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:08.108697] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF263-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:08.120623] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC27-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:08.123147] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC27-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:08.123167] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC27-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:08.183490] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5AB-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.195147] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEA96-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:08.213597] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF264-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:08.214847] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF264-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.214857] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF264-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:08.214873] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF264-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:08.214882] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF264-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:08.235962] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC45-0-0] [lt=34] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:08.287276] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5AC-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.312188] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEA96-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:08.319881] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF265-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:08.321142] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF265-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.321154] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF265-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:08.321170] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF265-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:08.321179] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF265-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:08.352177] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:08.390719] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5AD-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.426072] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF266-0-0] [lt=31] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:08.427291] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF266-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.427302] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF266-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:08.427318] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF266-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:08.427327] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF266-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:08.480735] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC48-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:08.494370] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5AE-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.494621] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5AE-0-0] [lt=51] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874426991810002, v:0}}], last_dump_ts_=1733118903471521, force_dump=false) [2024-12-02 13:55:08.532207] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF267-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:08.551884] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC28-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:08.554351] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC28-0-0] [lt=16] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:08.577353] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA7D-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:08.577970] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF267-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.577982] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF267-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:08.577998] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF267-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:08.578008] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF267-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:08.579755] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA7D-0-0] [lt=30] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:08.672348] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5AF-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.682844] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF268-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:08.684298] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF268-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.684307] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF268-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:08.684319] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF268-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:08.684327] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF268-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:08.776024] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5B0-0-0] [lt=50] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874408973130000, v:0}}], last_dump_ts_=1733118903678547, force_dump=false) [2024-12-02 13:55:08.776058] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5B0-0-0] [lt=8][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.789108] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF269-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:08.796782] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF269-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.796793] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF269-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:08.796809] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF269-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:08.796819] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF269-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:08.879641] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5B1-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.901547] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26A-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:08.903139] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26A-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.903150] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26A-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:08.903167] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26A-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:08.903172] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26A-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:08.944160] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC3A-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:08.995380] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5B2-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.008021] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26B-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:09.009233] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26B-0-0] [lt=14][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.009240] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26B-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:09.009256] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26B-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:09.009265] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:09.052202] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC42-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:09.063767] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB0-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:55:09.069598] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB0-0-0] [lt=20] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118908558064003, v:0}}], last_dump_ts_=1733118904009593, force_dump=false) [2024-12-02 13:55:09.069621] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB0-0-0] [lt=20] is primary tenant, no need check readable_scn [2024-12-02 13:55:09.098665] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5B3-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.114023] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26C-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:09.121639] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26C-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.121650] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26C-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:09.121665] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26C-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:09.121671] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26C-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:09.125872] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC28-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:09.128384] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC28-0-0] [lt=51] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:09.128402] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC28-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:09.195225] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEA97-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:09.201969] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5B4-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.226428] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26D-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:09.228751] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26D-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.228761] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26D-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:09.228776] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26D-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:09.228786] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26D-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:09.236074] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC46-0-0] [lt=43] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:09.283918] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320556][T1_FrzInfoDet][T1][YB42C0A800AF-00062843088EE997-0-0] [lt=20] get number(ret=0, service_type_=0, range=1, base_id=1733118909283081000, start_id=1733118909283081000, end_id=1733118909283081001) [2024-12-02 13:55:09.285668] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320556][T1_FrzInfoDet][T1][YB42C0A800AF-00062843088EE997-0-0] [lt=21] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118909283081000, v:0}, tenant_id=1) [2024-12-02 13:55:09.312259] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEA97-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:09.313956] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5B5-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.333584] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26E-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:09.334805] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26E-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.334815] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26E-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:09.334830] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26E-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:09.334835] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26E-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:09.352242] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:09.417364] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5B6-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.439712] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26F-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:09.441214] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26F-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.441225] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26F-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:09.441241] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26F-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:09.441250] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26F-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:09.480788] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC49-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:09.520873] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5B7-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.545989] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF270-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:09.547201] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF270-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.547212] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF270-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:09.547227] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF270-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:09.547236] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF270-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:09.559697] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC29-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:09.562197] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC29-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:09.586605] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA7E-0-0] [lt=23] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:09.589049] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA7E-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:09.620868] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319551][LeaseQueueTh2][T0][YB42C0A800AF-000628430B4EEAC2-0-0] [lt=16] [DDL] execute ddl like stmt(ret=0, cost=19, ddl_arg=NULL) [2024-12-02 13:55:09.624232] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5B8-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.651959] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF271-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:09.653448] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF271-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.653459] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF271-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:09.653482] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF271-0-0] [lt=22][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:09.653488] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF271-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:09.728083] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5B9-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.758278] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF272-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:09.759822] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF272-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.759832] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF272-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:09.759848] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF272-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:09.759857] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF272-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:09.831706] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5BA-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.848969] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAC8-0-0] [lt=21] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:09.864624] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF273-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:09.873116] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF273-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.873127] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF273-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:09.873142] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF273-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:09.873151] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF273-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:09.935503] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5BB-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.945556] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC3B-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:09.977953] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF274-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:09.979468] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF274-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.979479] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF274-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:09.979494] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF274-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:09.979500] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF274-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:10.031064] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAC4-0-0] [lt=14] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:10.038885] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5BC-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.052263] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC43-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:10.075072] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB1-0-0] [lt=9] is primary tenant, no need check readable_scn [2024-12-02 13:55:10.080885] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB1-0-0] [lt=15] is primary tenant, no need check readable_scn [2024-12-02 13:55:10.084340] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF275-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:10.085587] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF275-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.085597] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF275-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:10.085613] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF275-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:10.085619] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF275-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:10.131095] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC29-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:10.133521] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC29-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:10.133539] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC29-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:10.142460] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5BD-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.190550] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF276-0-0] [lt=20] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:10.192114] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF276-0-0] [lt=10][errcode=-4018] REACH SYSLOG RATE LIMIT [bandwidth] [2024-12-02 13:55:10.192129] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF276-0-0] [lt=0] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:10.195302] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEA98-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:10.236173] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC47-0-0] [lt=38] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:10.246038] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5BE-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.296843] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF277-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:10.298176] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF277-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.298186] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF277-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:10.298202] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF277-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:10.298207] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF277-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:10.312331] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEA98-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:10.349636] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5BF-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.352096] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118908558064004, replayable_scn:1733118908558064004, readable_scn:1733118908558064003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118910348801, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118908558064004, replayable_scn:1733118908558064004, readable_scn:1733118908558064003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118910348801, ora_rowscn_=1733118910077385000, content_changed=true) [2024-12-02 13:55:10.352320] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=26] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:10.352337] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=213, return_code_array=[0], broadcast_times=1) [2024-12-02 13:55:10.403059] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF278-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:10.404660] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF278-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.404671] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF278-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:10.404687] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF278-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:10.404697] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF278-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:10.406515] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=12] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118910348801, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118908558064004, replayable_scn:1733118908558064004, readable_scn:1733118908558064003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:55:10.454170] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5C0-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.480863] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC4A-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:10.509603] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF279-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:10.510827] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF279-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.510834] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF279-0-0] [lt=7][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:10.510851] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF279-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:10.510856] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF279-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:10.527278] INFO [RS] run2 (ob_archive_scheduler_service.cpp:85) [2320162][T1001_ArchiveSv][T1001][YB42C0A800AF-000628430ACEE95C-0-0] [lt=10] start do ObArchiveSchedulerService round(round=17) [2024-12-02 13:55:10.530597] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:954) [2320162][T1001_ArchiveSv][T1001][YB42C0A800AF-000628430ACEE95C-0-0] [lt=17][errcode=-4018] no row exist(ret=-4018) [2024-12-02 13:55:10.530614] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:627) [2320162][T1001_ArchiveSv][T1001][YB42C0A800AF-000628430ACEE95C-0-0] [lt=16][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-12-02 13:55:10.530672] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [2320162][T1001_ArchiveSv][T1001][YB42C0A800AF-000628430ACEE95C-0-0] [lt=11][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-12-02 13:55:10.530689] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [2320162][T1001_ArchiveSv][T1001][YB42C0A800AF-000628430ACEE95C-0-0] [lt=16][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-12-02 13:55:10.530701] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [2320162][T1001_ArchiveSv][T1001][YB42C0A800AF-000628430ACEE95C-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-12-02 13:55:10.559251] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5C1-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.567475] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC2A-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:10.569909] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC2A-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:10.595871] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA7F-0-0] [lt=23] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:10.598338] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA7F-0-0] [lt=26] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:10.615766] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27A-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:10.617400] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27A-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.617412] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27A-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:10.617428] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27A-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:10.617437] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27A-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:10.663055] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5C2-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.722133] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27B-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:10.723392] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27B-0-0] [lt=14][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.723402] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27B-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:10.723418] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27B-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:10.723427] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:10.766731] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5C3-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.828236] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27C-0-0] [lt=20] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:10.829736] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27C-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.829747] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27C-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:10.829763] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27C-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:10.829772] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27C-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:10.853414] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118910850718, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118910850718, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:55:10.870409] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5C4-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.934550] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27D-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:10.935781] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27D-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.935792] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27D-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:10.935808] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27D-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:10.935816] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27D-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:10.946949] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC3C-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:10.954715] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1020, epoch_=0, all_service_names_=[]) [2024-12-02 13:55:10.973978] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5C5-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.040778] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27E-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:11.042392] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27E-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.042403] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27E-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:11.042420] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27E-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:11.042426] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27E-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:11.052339] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC44-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:11.077391] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5C6-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.085793] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB2-0-0] [lt=17] is primary tenant, no need check readable_scn [2024-12-02 13:55:11.091061] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB2-0-0] [lt=31] is primary tenant, no need check readable_scn [2024-12-02 13:55:11.132807] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:11.134226] WDIAG [RS] get_dbms_job_infos_in_tenant (ob_dbms_job_utils.cpp:363) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=4][errcode=-5019] execute query failed(ret=-5019, sql=select * from __all_job, tenant_id=1020) [2024-12-02 13:55:11.134283] WDIAG [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:613) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=9][errcode=-5019] fail to exec job_utils_.get_dbms_job_infos_in_tenant(tenant_id, allocator, job_infos)(ret=-5019) [2024-12-02 13:55:11.134294] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10] load and register new jobs(ret=-5019, job_key=NULL, job_key=NULL, job_infos=[]) [2024-12-02 13:55:11.134305] WDIAG [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:590) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10][errcode=-5019] fail to exec load_and_register_new_jobs(tenant_ids.at(i))(ret=-5019) [2024-12-02 13:55:11.134317] INFO [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:593) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] register single tenant(ret=-5019, tenant_ids.at(i)=1020, tenant_schema->get_compatibility_mode()=0) [2024-12-02 13:55:11.134328] WDIAG [RS] scheduler_job (ob_dbms_job_master.cpp:368) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=9][errcode=-5019] fail to exec load_and_register_all_jobs(job_key)(ret=-5019) [2024-12-02 13:55:11.134336] WDIAG [RS] scheduler (ob_dbms_job_master.cpp:340) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7][errcode=0] fail to scheduler single dbms job(ret=0, tmp_ret=-5019, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1733118916132825, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:55:11.136233] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2A-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:11.138792] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2A-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:11.138811] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2A-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:11.147233] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27F-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:11.148495] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27F-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.148506] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27F-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:11.148522] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27F-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:11.148531] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27F-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:11.182031] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5C7-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.195376] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEA99-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:11.210276] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:11.211434] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=16] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118911210211, delay:5000000, job_id:0, tenant_id:1019}, job_key=0xfffeb4af6120, job_infos=[]) [2024-12-02 13:55:11.211457] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=20] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118916211455, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:55:11.214515] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2324931][T1019_FrzInfoDe][T1019][YB42C0A800AF-00062843139EE96D-0-0] [lt=44] get number(ret=0, service_type_=0, range=1, base_id=1733118911213644000, start_id=1733118911213644000, end_id=1733118911213644001) [2024-12-02 13:55:11.216181] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2324931][T1019_FrzInfoDe][T1019][YB42C0A800AF-00062843139EE96D-0-0] [lt=17] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118911213644000, v:0}, tenant_id=1019) [2024-12-02 13:55:11.236273] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC48-0-0] [lt=35] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:11.253446] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF280-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:11.254769] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF280-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.254780] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF280-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:11.254796] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF280-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:11.254805] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF280-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:11.285347] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5C8-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.305187] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:11.306342] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=17] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118911305123, delay:5000000, job_id:0, tenant_id:1001}, job_key=0xfffeb4af60a0, job_infos=[]) [2024-12-02 13:55:11.306370] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=26] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118916306369, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-02 13:55:11.307481] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118911306277, delay:5000000, job_id:0, tenant_id:1002}, job_key=0xfffeb4af60e0, job_infos=[]) [2024-12-02 13:55:11.307505] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=22] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118916307503, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-02 13:55:11.312402] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEA99-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:11.352334] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:11.353677] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=11] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-02 13:55:11.359628] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF281-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:11.361095] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF281-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.361102] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF281-0-0] [lt=7][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:11.361118] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF281-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:11.361123] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF281-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:11.388559] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5C9-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.465944] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF282-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:11.467453] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF282-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.467464] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF282-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:11.467479] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF282-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:11.467489] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF282-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:11.480940] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC4B-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:11.492276] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5CA-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.572342] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF283-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:11.573948] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF283-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.573959] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF283-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:11.573975] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF283-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:11.573984] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF283-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:11.575292] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC2B-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:11.577771] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC2B-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:11.595660] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5CB-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.605084] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA80-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:11.607529] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA80-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:11.621302] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319549][LeaseQueueTh0][T0][YB42C0A800AF-000628430B4EEAC3-0-0] [lt=16] [DDL] execute ddl like stmt(ret=0, cost=13, ddl_arg=NULL) [2024-12-02 13:55:11.678704] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF284-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:11.681011] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF284-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.681022] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF284-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:11.681038] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF284-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:11.681047] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF284-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:11.699626] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5CC-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.785755] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF285-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:11.803005] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5CD-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.804949] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF285-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.804959] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF285-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:11.804975] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF285-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:11.804984] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF285-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:11.853775] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAC9-0-0] [lt=23] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:11.906924] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5CE-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.909779] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF286-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:11.911356] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF286-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.911366] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF286-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:11.911382] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF286-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:11.911392] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF286-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:11.948318] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC3D-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:12.010506] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5CF-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.016173] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF287-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:12.018224] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF287-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.018236] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF287-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:12.018249] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF287-0-0] [lt=11][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:12.018258] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF287-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:12.031376] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAC5-0-0] [lt=13] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:12.052414] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC45-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:12.096452] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB3-0-0] [lt=14] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118911708100002, v:0}}], last_dump_ts_=1733118907043702, force_dump=false) [2024-12-02 13:55:12.096475] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB3-0-0] [lt=20] is primary tenant, no need check readable_scn [2024-12-02 13:55:12.101765] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB3-0-0] [lt=9] is primary tenant, no need check readable_scn [2024-12-02 13:55:12.114216] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5D0-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.122955] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF288-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:12.124183] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF288-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.124193] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF288-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:12.124209] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF288-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:12.124218] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF288-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:12.141556] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2B-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:12.144110] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2B-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:12.144130] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2B-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:12.195453] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEA9A-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:12.218546] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5D1-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.228919] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF289-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:12.230116] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF289-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.230127] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF289-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:12.230143] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF289-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:12.230152] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF289-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:12.236364] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC49-0-0] [lt=31] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:12.312474] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEA9A-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:12.322100] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5D2-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.334903] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28A-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:12.336099] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28A-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.336106] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28A-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:12.336122] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28A-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:12.336131] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28A-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:12.352322] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:12.425619] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5D3-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.440803] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28B-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:12.442054] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28B-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.442064] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28B-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:12.442076] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28B-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:12.442084] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:12.481017] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC4C-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:12.529285] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5D4-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.546818] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28C-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:12.554150] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28C-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.554161] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28C-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:12.554177] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28C-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:12.554185] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28C-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:12.583198] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC2C-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:12.585674] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC2C-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:12.614304] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA81-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:12.616702] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA81-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:12.632682] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5D5-0-0] [lt=22][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.658974] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28D-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:12.660312] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28D-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.660323] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28D-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:12.660338] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28D-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:12.660347] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28D-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:12.736287] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5D6-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.765219] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28E-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:12.766535] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28E-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.766546] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28E-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:12.766563] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28E-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:12.766572] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28E-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:12.840443] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5D7-0-0] [lt=21][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.871663] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28F-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:12.873175] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28F-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.873186] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28F-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:12.873202] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28F-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:12.873211] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28F-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:12.944098] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5D8-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.949724] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC3E-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:12.975547] INFO [SHARE] adaptive_update_limit (ob_share_throttle_define.cpp:112) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE997-0-0] [lt=20] adaptive update(Tenant ID=1001, Config Specify Resource Limit(MB)=511, TxShare Current Memory Limit(MB)=360, Holding Memory(MB)=61, Tenant Remain Memory(MB)=497, Usable Remain Memory(MB)=298, Last Update Limit Timestamp=1733118912975018, Is Updated=true) [2024-12-02 13:55:12.975718] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE997-0-0] [lt=23] get number(ret=0, service_type_=0, range=1, base_id=1733118912975018000, start_id=1733118912975018000, end_id=1733118912975018001) [2024-12-02 13:55:12.978282] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF290-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:12.978583] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE997-0-0] [lt=20] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118912975018000, v:0}, tenant_id=1001) [2024-12-02 13:55:12.980520] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF290-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.980531] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF290-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:12.980547] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF290-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:12.980557] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF290-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:13.047860] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5D9-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:13.052489] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC46-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:13.085540] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF291-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:13.087288] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF291-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:13.087298] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF291-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:13.087314] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF291-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:13.087324] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF291-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:13.112934] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB4-0-0] [lt=15] is primary tenant, no need check readable_scn [2024-12-02 13:55:13.119503] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB4-0-0] [lt=18] is primary tenant, no need check readable_scn [2024-12-02 13:55:13.146825] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2C-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:13.149351] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2C-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:13.149371] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2C-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:13.151991] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5DA-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:13.192297] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF292-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:13.193928] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF292-0-0] [lt=11][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:13.193939] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF292-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:13.193955] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF292-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:13.193964] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF292-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:13.195527] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEA9B-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:13.236458] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC4A-0-0] [lt=27] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:13.255755] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5DB-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:13.281309] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320099][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDC4-0-0] [lt=17] [DDL] execute ddl like stmt(ret=0, cost=10, ddl_arg=NULL) [2024-12-02 13:55:13.286176] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320107][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDC5-0-0] [lt=10] [DDL] execute ddl like stmt(ret=0, cost=10, ddl_arg=NULL) [2024-12-02 13:55:13.288930] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320104][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDC9-0-0] [lt=12] [DDL] execute ddl like stmt(ret=0, cost=8, ddl_arg=NULL) [2024-12-02 13:55:13.289138] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320104][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDCA-0-0] [lt=15] [DDL] execute ddl like stmt(ret=0, cost=4, ddl_arg=NULL) [2024-12-02 13:55:13.289760] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320105][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDCB-0-0] [lt=9] [DDL] execute ddl like stmt(ret=0, cost=7, ddl_arg=NULL) [2024-12-02 13:55:13.289949] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320105][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDCC-0-0] [lt=11] [DDL] execute ddl like stmt(ret=0, cost=3, ddl_arg=NULL) [2024-12-02 13:55:13.290140] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320104][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDCD-0-0] [lt=17] [DDL] execute ddl like stmt(ret=0, cost=5, ddl_arg=NULL) [2024-12-02 13:55:13.290882] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320104][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDD3-0-0] [lt=3] [DDL] execute ddl like stmt(ret=0, cost=5, ddl_arg=NULL) [2024-12-02 13:55:13.291426] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320108][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDD4-0-0] [lt=9] [DDL] execute ddl like stmt(ret=0, cost=11, ddl_arg=NULL) [2024-12-02 13:55:13.295171] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:452) [2320369][T1002_TntShared][T1002][Y0-0000000000000000-0-0] [lt=19] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2024-12-02 13:55:13.299390] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF293-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:13.301237] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF293-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:13.301249] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF293-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:13.301267] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF293-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:13.301276] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF293-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:13.312546] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEA9B-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:13.352303] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118911708100002, replayable_scn:1733118911708100002, readable_scn:1733118911708100002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118913348583, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118911708100002, replayable_scn:1733118911708100002, readable_scn:1733118911708100002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118913348583, ora_rowscn_=1733118913114919001, content_changed=true) [2024-12-02 13:55:13.352584] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=22] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:13.359619] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5DC-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:13.387398] INFO [RS] finish (ob_mview_maintenance_task.cpp:282) [2320369][T1002_TntShared][T1002][Y0-0000000000000000-0-0] [lt=32] mview maintenace task finish(tenant_id_=1002, round_=10, status_=100, error_code_=0, cost_us_=20001321, prepare_cost_us_=1, gc_mview_cost_us_=1157, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2024-12-02 13:55:13.406441] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF294-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:13.408189] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=23] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118913348583, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118911708100002, replayable_scn:1733118911708100002, readable_scn:1733118911708100002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:55:13.408934] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF294-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:13.408945] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF294-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:13.408963] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF294-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:13.408972] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF294-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:13.463462] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5DD-0-0] [lt=19][errcode=0] REACH SYSLOG RATE LIMIT [bandwidth] [2024-12-02 13:55:13.468265] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320098][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDD6-0-0] [lt=8] [DDL] execute ddl like stmt(ret=0, cost=9, ddl_arg=NULL) [2024-12-02 13:55:13.481095] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC4D-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:13.513900] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF295-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:13.515794] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF295-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:13.515804] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF295-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:13.515817] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF295-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:13.515825] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF295-0-0] [lt=6] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:13.566167] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5DE-0-0] [lt=0] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874426991810002, v:0}}], last_dump_ts_=1733118908494621, force_dump=false) [2024-12-02 13:55:13.591042] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC2D-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:13.593497] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC2D-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:13.620828] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF296-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:13.621779] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319549][LeaseQueueTh0][T0][YB42C0A800AF-000628430B4EEAC4-0-0] [lt=20] [DDL] execute ddl like stmt(ret=0, cost=17, ddl_arg=NULL) [2024-12-02 13:55:13.622469] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF296-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:13.622479] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF296-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:13.622496] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF296-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:13.622505] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF296-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:13.623363] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA82-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:13.626684] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA82-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:13.685386] INFO [RS.LB] idle (ob_root_balancer.cpp:103) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC1F-0-0] [lt=19] root balance idle(idle_time=10000000) [2024-12-02 13:55:13.685415] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2074) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=20] start try disaster recovery [2024-12-02 13:55:13.685427] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1, only_for_display=false) [2024-12-02 13:55:13.686677] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=10] start try ls disaster recovery(ls_status_info={tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:13.691243] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=25] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:13.691264] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=16] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.691271] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:1733118159456434, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:13.691322] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=47] build ls locality stat map(tenant_id=1, ls_id={id:1}) [2024-12-02 13:55:13.691331] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:13.691341] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118159456434, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:13.691373] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=24] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.691385] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.691392] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.691396] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.693356] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=4] try tenant disaster recovery succ(tenant_id=1, only_for_display=false) [2024-12-02 13:55:13.693367] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1001, only_for_display=false) [2024-12-02 13:55:13.694654] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1001, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:13.699111] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1001, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:13.699132] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=16] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.699139] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=5] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1001, ls_id:{id:1}, replicas:[{modify_time_us:1733118160462607, create_time_us:1732871267341550, tenant_id:1001, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1001, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:13.699188] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=46] build ls locality stat map(tenant_id=1001, ls_id={id:1}) [2024-12-02 13:55:13.699197] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:13.699207] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118160462607, create_time_us:1732871267341550, tenant_id:1001, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:13.699239] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=24] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.699251] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.699257] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=4] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.699262] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.701045] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=3] try tenant disaster recovery succ(tenant_id=1001, only_for_display=false) [2024-12-02 13:55:13.702360] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=8] start try tenant disaster recovery(tenant_id=1002, only_for_display=false) [2024-12-02 13:55:13.703584] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=21] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:13.708006] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=27] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:13.708036] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=24] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.708047] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1}, replicas:[{modify_time_us:1733118167177128, create_time_us:1732871280733688, tenant_id:1002, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:13.708100] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=50] build ls locality stat map(tenant_id=1002, ls_id={id:1}) [2024-12-02 13:55:13.708113] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:13.708126] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118167177128, create_time_us:1732871280733688, tenant_id:1002, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:13.708160] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.708175] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.708184] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.708192] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.710167] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:13.716952] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=21] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1001}, acc_dr_task=0) [2024-12-02 13:55:13.716982] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=24] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.716993] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1001}, replicas:[{modify_time_us:1733118167179939, create_time_us:1732871299864461, tenant_id:1002, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:13.717045] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=50] build ls locality stat map(tenant_id=1002, ls_id={id:1001}) [2024-12-02 13:55:13.717058] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:13.717070] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118167179939, create_time_us:1732871299864461, tenant_id:1002, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:13.717104] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.717118] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.717126] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.717134] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.719065] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=7] try tenant disaster recovery succ(tenant_id=1002, only_for_display=false) [2024-12-02 13:55:13.719085] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=19] start try tenant disaster recovery(tenant_id=1019, only_for_display=false) [2024-12-02 13:55:13.720331] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1019, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:13.724651] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=25] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1019, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:13.724678] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=22] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.724687] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1019, ls_id:{id:1}, replicas:[{modify_time_us:1733118580950623, create_time_us:1733118578521026, tenant_id:1019, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1019, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:13.724738] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=47] build ls locality stat map(tenant_id=1019, ls_id={id:1}) [2024-12-02 13:55:13.724749] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:13.724761] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118580950623, create_time_us:1733118578521026, tenant_id:1019, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:13.724794] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=26] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.724811] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.724818] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.724825] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.727116] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=4] try tenant disaster recovery succ(tenant_id=1019, only_for_display=false) [2024-12-02 13:55:13.727699] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF297-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:13.728298] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=17] start try tenant disaster recovery(tenant_id=1020, only_for_display=false) [2024-12-02 13:55:13.729351] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF297-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:13.729362] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF297-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:13.729378] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF297-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:13.729387] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF297-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:13.729484] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=21] start try ls disaster recovery(ls_status_info={tenant_id:1020, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:13.733852] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=24] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1020, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:13.733882] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=24] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.733893] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1020, ls_id:{id:1}, replicas:[{modify_time_us:1733118656791956, create_time_us:1733118591069170, tenant_id:1020, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1020, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:13.733946] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=50] build ls locality stat map(tenant_id=1020, ls_id={id:1}) [2024-12-02 13:55:13.733958] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:13.733971] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118656791956, create_time_us:1733118591069170, tenant_id:1020, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:13.734006] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.734021] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.734030] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.734039] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.735922] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1020, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1010, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:13.742646] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=21] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1020, ls_id={id:1001}, acc_dr_task=0) [2024-12-02 13:55:13.742676] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=24] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.742687] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1020, ls_id:{id:1001}, replicas:[{modify_time_us:1733118655283621, create_time_us:1733118641058857, tenant_id:1020, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1020, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1010, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:13.742739] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=48] build ls locality stat map(tenant_id=1020, ls_id={id:1001}) [2024-12-02 13:55:13.742748] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:13.742761] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118655283621, create_time_us:1733118641058857, tenant_id:1020, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:13.742793] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=26] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.742808] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.742817] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.742825] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.744674] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=7] try tenant disaster recovery succ(tenant_id=1020, only_for_display=false) [2024-12-02 13:55:13.744692] INFO [RS] check (ob_migrate_unit_finish_checker.cpp:79) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=15] start check unit migrate finish [2024-12-02 13:55:13.744706] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=5] try check migrate unit finish by tenant(tenant_id=1) [2024-12-02 13:55:13.750573] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=9] try check migrate unit finish by tenant(tenant_id=1002) [2024-12-02 13:55:13.760902] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=18] try check migrate unit finish by tenant(tenant_id=1020) [2024-12-02 13:55:13.771035] INFO [RS] check (ob_alter_locality_finish_checker.cpp:79) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=19] start to check alter locality finish [2024-12-02 13:55:13.772157] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC21-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:13.773257] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC21-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:13.775566] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC23-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:13.776669] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC23-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:13.778927] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC25-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:13.780047] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC25-0-0] [lt=19][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:13.781250] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:72) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC25-0-0] [lt=18] start check shrink resource pool [2024-12-02 13:55:13.781271] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC26-0-0] [lt=16] start check shrink resource pool(tenant_id=1) [2024-12-02 13:55:13.782340] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC26-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:13.783437] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC26-0-0] [lt=17][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:13.783515] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC27-0-0] [lt=18] start check shrink resource pool(tenant_id=1002) [2024-12-02 13:55:13.784573] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC27-0-0] [lt=15][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:13.784627] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5E0-0-0] [lt=1] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874408973130000, v:0}}], last_dump_ts_=1733118908776022, force_dump=false) [2024-12-02 13:55:13.785688] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC27-0-0] [lt=16][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:13.785764] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC28-0-0] [lt=16] start check shrink resource pool(tenant_id=1020) [2024-12-02 13:55:13.786815] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC28-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:13.787913] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC28-0-0] [lt=17][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:13.787990] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:100) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC28-0-0] [lt=17] finish check shrink resource pool(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:13.788006] INFO [RS] check (ob_alter_primary_zone_checker.cpp:49) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC28-0-0] [lt=14] start check primary zone [2024-12-02 13:55:13.788018] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC29-0-0] [lt=7] start check primary zone(tenant_id=1) [2024-12-02 13:55:13.789056] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC29-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:13.789136] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2A-0-0] [lt=17] start check primary zone(tenant_id=1002) [2024-12-02 13:55:13.790194] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2A-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:13.790271] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=16] start check primary zone(tenant_id=1020) [2024-12-02 13:55:13.791306] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:13.791384] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:179) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=17] start do unit balance [2024-12-02 13:55:13.793836] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:363) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=13] start distribute for server status change [2024-12-02 13:55:13.793889] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:414) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=17] finish distribute for server status change(ret=0) [2024-12-02 13:55:13.793899] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:213) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=10] finish do unit balance(ret=0) [2024-12-02 13:55:13.796211] INFO [RS.LB] tenant_group_balance (ob_server_balancer.cpp:128) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=7] start to do tenant group unit balance [2024-12-02 13:55:13.796259] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:930) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=17] start to do tenant group unit balance(zone=zone1) [2024-12-02 13:55:13.796276] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7935) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=11] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone1", server_disk_statistic_array_:[{server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}], over_disk_waterlevel_:false}) [2024-12-02 13:55:13.796295] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1084) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=13] generate tenant array finish(ret=0, zone=zone1, standalone_tenant_array=[1001, 1002, 1019, 1020], group_tenant_array=[]) [2024-12-02 13:55:13.796319] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=11] generate available servers finish(ret=0, zone=zone1, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:55:13.796339] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=10] generate available servers finish(ret=0, zone=zone1, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:55:13.796350] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5311) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=8] start do non-tenantgroup unit balance task(zone=zone1) [2024-12-02 13:55:13.796361] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5640) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=7] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone1, cpu_weights=3.484898771995591504e-01, mem_weights=3.982741453709247037e-01, log_disk_weights=2.532359774295159793e-01, data_disk_weights=0.000000000000000000e+00, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:55:13.796384] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4547) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=16] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"192.168.0.175:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:12, max_cpu:12, memory_size:"12GB", log_disk_size:"109GB", data_disk_size:0, min_iops:9998, max_iops:9998, iops_weight:4, max_net_bandwidth:"-3.7252903e-09GB", net_bandwidth_weight:12, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:12, cpu_assigned_max:12, mem_capacity:"14GB", mem_assigned:"12GB", mem_in_use:0GB, log_disk_capacity:200GB, log_disk_assigned:109GB, log_disk_in_use:5.75GB, data_disk_capacity:10000GB, data_disk_assigned:0GB, data_disk_in_use:0.44140625GB}, inter_ttg_load_value_:"7.407588544869403435e-01"}], available_servers=["192.168.0.175:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"192.168.0.175:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:12, max_cpu:12, memory_size:"12GB", log_disk_size:"109GB", data_disk_size:0, min_iops:9998, max_iops:9998, iops_weight:4, max_net_bandwidth:"-3.7252903e-09GB", net_bandwidth_weight:12, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:12, cpu_assigned_max:12, mem_capacity:"14GB", mem_assigned:"12GB", mem_in_use:0GB, log_disk_capacity:200GB, log_disk_assigned:109GB, log_disk_in_use:5.75GB, data_disk_capacity:10000GB, data_disk_assigned:0GB, data_disk_in_use:0.44140625GB}, inter_ttg_load_value_:"7.407588544869403435e-01"}]) [2024-12-02 13:55:13.796422] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8126) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=37] check server over disk waterlevel(zone=zone1, disk_statistic={server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}, over_disk_waterlevel=false) [2024-12-02 13:55:13.796432] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8132) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=10] check all available servers over disk waterlevel(zone=zone1, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}]) [2024-12-02 13:55:13.796447] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5293) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=10] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=2, not_grant_units.count()=0) [2024-12-02 13:55:13.796459] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5348) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=10] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone1, upper_lmt=7.777967972112873607e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1001, tenant_id:1002, resource_pool_id:1001, unit_config:{unit_config_id:1001, name:"ocp_meta_unit", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0xfffe9c877ce8, unit_config:0xfffe9c865d18, pool:0xfffe9c883688}, {unit_id:1003, tenant_id:1020, resource_pool_id:1004, unit_config:{unit_config_id:1003, name:"hy_unit_config", resource:{min_cpu:8, max_cpu:8, memory_size:"8GB", log_disk_size:"100GB", data_disk_size:0, min_iops:10000, max_iops:10000, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:8, }}, unit:0xfffe9c877c10, unit_config:0xfffe9c865c38, pool:0xfffe9c8832d0}], not_grant_units=[], available_servers=["192.168.0.175:2882"]) [2024-12-02 13:55:13.834422] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF298-0-0] [lt=23] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:13.836120] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF298-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:13.836131] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF298-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:13.836147] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF298-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:13.836155] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF298-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:13.855096] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118913852503, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118913852503, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:55:13.858521] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEACA-0-0] [lt=19] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:13.941110] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF299-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:13.943058] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF299-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:13.943068] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF299-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:13.943084] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF299-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:13.943093] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF299-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:13.951139] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC3F-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:14.031673] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAC6-0-0] [lt=14] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:14.048139] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29A-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:14.049899] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29A-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:14.049908] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29A-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:14.049923] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29A-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:14.049931] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29A-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:14.052565] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC47-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:14.125619] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB5-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:55:14.133894] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB5-0-0] [lt=18] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118914077648001, v:0}}], last_dump_ts_=1733118909069596, force_dump=false) [2024-12-02 13:55:14.133916] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB5-0-0] [lt=19] is primary tenant, no need check readable_scn [2024-12-02 13:55:14.138771] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5E3-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:14.152066] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2D-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:14.154623] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2D-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:14.154643] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2D-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:14.154970] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29B-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:14.155928] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29B-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:14.155936] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29B-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:14.155951] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29B-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:14.155960] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:14.195599] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEA9C-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:14.198247] INFO [RS] remove_inactive_ddl_task (ob_ddl_scheduler.cpp:2588) [2320739][HeartBeatCheckT][T0][Y0-0000000000000000-0-0] [lt=5] need remove task(remove_task_ids=[]) [2024-12-02 13:55:14.198264] INFO [RS] remove_inactive_ddl_task (ob_ddl_scheduler.cpp:2607) [2320739][HeartBeatCheckT][T0][Y0-0000000000000000-0-0] [lt=16] remove all timeout ddl task succeed [2024-12-02 13:55:14.236557] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC4B-0-0] [lt=33] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:14.242389] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5E4-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:14.260992] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29C-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:14.262611] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29C-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:14.262621] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29C-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:14.262635] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29C-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:14.262642] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29C-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:14.312626] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEA9C-0-0] [lt=22] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:14.346090] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5E5-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:14.352443] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:14.352459] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=241, return_code_array=[0], broadcast_times=1) [2024-12-02 13:55:14.367638] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29D-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:14.369594] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29D-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:14.369601] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29D-0-0] [lt=7][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:14.369616] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29D-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:14.369625] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29D-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:14.474595] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29E-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:14.475877] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29E-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:14.475887] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29E-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:14.475902] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29E-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:14.475907] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29E-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:14.481171] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC4E-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:14.556330] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5E7-0-0] [lt=17][errcode=0] REACH SYSLOG RATE LIMIT [bandwidth] [2024-12-02 13:55:14.580884] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29F-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:14.583372] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29F-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:14.583383] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29F-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:14.583395] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29F-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:14.583404] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29F-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:14.598859] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC2E-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:14.601300] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC2E-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:14.633277] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA83-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:14.635670] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA83-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:14.688277] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A0-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:14.690017] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A0-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:14.690028] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A0-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:14.690044] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A0-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:14.690053] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A0-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:14.794974] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A1-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:14.796309] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A1-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:14.796319] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A1-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:14.796335] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A1-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:14.796344] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A1-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:14.901223] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A2-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:14.903138] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A2-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:14.903149] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A2-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:14.903165] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A2-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:14.903170] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A2-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:14.952485] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC40-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:15.008037] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A3-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:15.009251] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A3-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.009262] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A3-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:15.009277] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A3-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:15.009286] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A3-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:15.052643] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC48-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:15.114203] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A4-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:15.115789] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A4-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.115800] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A4-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:15.115816] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A4-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:15.115825] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A4-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:15.139539] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB6-0-0] [lt=8] is primary tenant, no need check readable_scn [2024-12-02 13:55:15.149850] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB6-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:55:15.157360] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2E-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:15.159921] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2E-0-0] [lt=52] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:15.159940] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2E-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:15.179056] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5ED-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.195674] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEA9D-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:15.220692] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A5-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:15.222302] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A5-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.222312] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A5-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:15.222329] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A5-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:15.222334] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A5-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:15.230369] INFO [SHARE] update_decay_factor_ (ob_throttle_unit.ipp:284) [2320687][T1002_FrzInfoDe][T1002][YB42C0A800AF-000628430B6EE997-0-0] [lt=18] [Throttle] Update Throttle Unit Config(is_adaptive_update=true, N=5.980047273635864258e+01, this=0xffff2773b208, enable_adaptive_limit_=true, Unit Name=TxShare, Config Specify Resource Limit(MB)=511, Resource Limit(MB)=299, Throttle Trigger(MB)=179, Throttle Percentage=60, Max Duration(us)=7200000000, Decay Factor=2.178559058567542252e+03) [2024-12-02 13:55:15.230562] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320687][T1002_FrzInfoDe][T1002][YB42C0A800AF-000628430B6EE997-0-0] [lt=32] get number(ret=0, service_type_=0, range=1, base_id=1733118915229908000, start_id=1733118915229908000, end_id=1733118915229908001) [2024-12-02 13:55:15.232617] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320687][T1002_FrzInfoDe][T1002][YB42C0A800AF-000628430B6EE997-0-0] [lt=20] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118915229908000, v:0}, tenant_id=1002) [2024-12-02 13:55:15.236654] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC4C-0-0] [lt=30] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:15.283188] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5EE-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.312702] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEA9D-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:15.327338] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A6-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:15.352493] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:15.360411] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A6-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.360421] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A6-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:15.360436] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A6-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:15.360444] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A6-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:15.386549] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5EF-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.465432] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A7-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:15.475897] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A7-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.475907] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A7-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:15.475923] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A7-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:15.475931] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A7-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:15.481246] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC4F-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:15.490203] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5F0-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.580942] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A8-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:15.582545] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A8-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.582556] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A8-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:15.582568] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A8-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:15.582573] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A8-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:15.593599] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5F1-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.606514] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC2F-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:15.608925] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC2F-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:15.622192] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319549][LeaseQueueTh0][T0][YB42C0A800AF-000628430B4EEAC5-0-0] [lt=13] [DDL] execute ddl like stmt(ret=0, cost=14, ddl_arg=NULL) [2024-12-02 13:55:15.642377] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA84-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:15.644782] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA84-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:15.687502] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A9-0-0] [lt=14] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:15.689027] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A9-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.689038] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A9-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:15.689055] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A9-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:15.689064] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A9-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:15.708684] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5F2-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.794285] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AA-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:15.795700] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AA-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.795712] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AA-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:15.795729] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AA-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:15.795740] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AA-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:15.812068] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5F3-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.863192] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEACB-0-0] [lt=21] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:15.900792] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AB-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:15.903197] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AB-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.903208] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AB-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:15.903223] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AB-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:15.903232] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AB-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:15.915781] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5F4-0-0] [lt=12][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.953832] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC41-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:15.957407] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1020, epoch_=0, all_service_names_=[]) [2024-12-02 13:55:16.008117] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AC-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:16.009307] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AC-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.009317] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AC-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:16.009333] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AC-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:16.009342] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AC-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:16.025866] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5F5-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.031980] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAC7-0-0] [lt=18] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:16.052725] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC49-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:16.114340] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AD-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:16.116512] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AD-0-0] [lt=7][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.116519] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AD-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:16.116535] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AD-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:16.116544] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AD-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:16.129271] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5F6-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.132892] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:16.134285] WDIAG [RS] get_dbms_job_infos_in_tenant (ob_dbms_job_utils.cpp:363) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=4][errcode=-5019] execute query failed(ret=-5019, sql=select * from __all_job, tenant_id=1020) [2024-12-02 13:55:16.134337] WDIAG [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:613) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=5][errcode=-5019] fail to exec job_utils_.get_dbms_job_infos_in_tenant(tenant_id, allocator, job_infos)(ret=-5019) [2024-12-02 13:55:16.134348] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] load and register new jobs(ret=-5019, job_key=NULL, job_key=NULL, job_infos=[]) [2024-12-02 13:55:16.134359] WDIAG [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:590) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10][errcode=-5019] fail to exec load_and_register_new_jobs(tenant_ids.at(i))(ret=-5019) [2024-12-02 13:55:16.134367] INFO [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:593) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7] register single tenant(ret=-5019, tenant_ids.at(i)=1020, tenant_schema->get_compatibility_mode()=0) [2024-12-02 13:55:16.134378] WDIAG [RS] scheduler_job (ob_dbms_job_master.cpp:368) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=9][errcode=-5019] fail to exec load_and_register_all_jobs(job_key)(ret=-5019) [2024-12-02 13:55:16.134386] WDIAG [RS] scheduler (ob_dbms_job_master.cpp:340) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=0] fail to scheduler single dbms job(ret=0, tmp_ret=-5019, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1733118921132911, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:55:16.154769] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB7-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:55:16.162579] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2F-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:16.165073] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2F-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:16.165093] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2F-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:16.175199] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB7-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:55:16.195750] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEA9E-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:16.211518] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:16.212674] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=16] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118916211455, delay:5000000, job_id:0, tenant_id:1019}, job_key=0xfffeb4af6120, job_infos=[]) [2024-12-02 13:55:16.212700] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=24] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118921212698, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:55:16.221443] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AE-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:16.223101] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AE-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.223112] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AE-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:16.223128] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AE-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:16.223133] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AE-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:16.232961] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5F7-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.236759] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC4D-0-0] [lt=39] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:16.306433] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:16.307564] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=17] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118916306369, delay:5000000, job_id:0, tenant_id:1001}, job_key=0xfffeb4af60a0, job_infos=[]) [2024-12-02 13:55:16.307592] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=26] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118921307590, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-02 13:55:16.308679] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118916307503, delay:5000000, job_id:0, tenant_id:1002}, job_key=0xfffeb4af60e0, job_infos=[]) [2024-12-02 13:55:16.308697] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118921308695, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-02 13:55:16.312775] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEA9E-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:16.328079] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AF-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:16.329331] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AF-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.329341] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AF-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:16.329357] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AF-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:16.329365] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AF-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:16.337043] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5F8-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.352359] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118915064689000, replayable_scn:1733118915064689000, readable_scn:1733118914563953004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118916349261, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118915064689000, replayable_scn:1733118915064689000, readable_scn:1733118914563953004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118916349261, ora_rowscn_=1733118916156831001, content_changed=true) [2024-12-02 13:55:16.352608] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=23] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:16.353945] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-02 13:55:16.409820] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=15] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118916349261, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118915064689000, replayable_scn:1733118915064689000, readable_scn:1733118914563953004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:55:16.434340] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B0-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:16.436066] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B0-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.436077] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B0-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:16.436092] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B0-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:16.436101] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B0-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:16.440629] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5F9-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.481320] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC50-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:16.541066] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B1-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:16.542490] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B1-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.542501] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B1-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:16.542517] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B1-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:16.542526] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B1-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:16.556638] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5FA-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.614146] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC30-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:16.617601] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC30-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:16.647418] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B2-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:16.649379] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B2-0-0] [lt=0] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:16.651573] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA85-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:16.654000] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA85-0-0] [lt=26] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:16.660142] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5FB-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.754264] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B3-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:16.755652] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B3-0-0] [lt=0] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:16.759131] INFO [RS.COMPACTION] can_start_work (ob_freeze_info_detector.cpp:265) [2325689][T1020_FrzInfoDe][T1020][YB42C0A800AF-00062843179EE966-0-0] [lt=26] tenant is in abnormal status, no need detect now(tenant_id=1020, tenant_schema={tenant_id:1020, schema_version:1733118578035224, tenant_name:"mysql_hy", name_case_mode:-1, read_only:false, primary_zone:"RANDOM", locality:"FULL{1}@zone1", previous_locality:"", compatibility_mode:0, gmt_modified:1733118578035236, drop_tenant_time:-1, status:3, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:16.766155] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5FC-0-0] [lt=11][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.856683] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118916853341, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118916853341, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:55:16.860406] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B4-0-0] [lt=34] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:16.861978] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B4-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.861988] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B4-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:16.862004] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B4-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:16.862013] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B4-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:16.869552] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5FD-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.939146] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE966-0-0] [lt=33] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:16.940341] INFO [RS] do_ls_balance_task_ (ob_recovery_ls_service.cpp:1308) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE966-0-0] [lt=18] no need process balance task(tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}) [2024-12-02 13:55:16.941502] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE966-0-0] [lt=15] [LS_RECOVERY] finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, start_scn={val:18446744073709551615, v:3}, thread_idx=0, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:16.955186] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC42-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:16.966828] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B5-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:16.968883] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B5-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.968894] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B5-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:16.968910] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B5-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:16.968919] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B5-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:16.973616] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5FE-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.981130] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:464) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5FE-0-0] [lt=54] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.052803] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC4A-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:17.073737] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B6-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:17.075609] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B6-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.075615] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B6-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:17.075631] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B6-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:17.075640] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B6-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:17.081480] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5FF-0-0] [lt=24][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.101403] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:717) [2320160][T1001_BackupDat][T1001][YB42C0A800AF-000628430AAEE96A-0-0] [lt=17] [DATA_BACKUP]success get jobs(job_attrs=[]) [2024-12-02 13:55:17.167704] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC30-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:17.170186] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC30-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:17.170205] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC30-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:17.180360] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB8-0-0] [lt=13] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118916712394002, v:0}}], last_dump_ts_=1733118912096449, force_dump=false) [2024-12-02 13:55:17.180385] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB8-0-0] [lt=22] is primary tenant, no need check readable_scn [2024-12-02 13:55:17.180476] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B7-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:17.181742] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B7-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.181753] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B7-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:17.181769] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B7-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:17.181778] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B7-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:17.185235] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF600-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.195828] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEA9F-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:17.206366] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB8-0-0] [lt=22] is primary tenant, no need check readable_scn [2024-12-02 13:55:17.236864] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC4E-0-0] [lt=43] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:17.275158] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320099][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDD7-0-0] [lt=3] [DDL] execute ddl like stmt(ret=0, cost=8, ddl_arg=NULL) [2024-12-02 13:55:17.286760] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B8-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:17.288399] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B8-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.288410] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B8-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:17.288426] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B8-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:17.288435] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B8-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:17.288662] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF601-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.312848] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEA9F-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:17.352679] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:17.392097] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF602-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.393288] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B9-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:17.394571] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B9-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.394582] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B9-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:17.394598] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B9-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:17.394608] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B9-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:17.406914] INFO [RS] check_lost_replicas (ob_lost_replica_checker.cpp:117) [2319631][LostRepCheck][T0][Y0-0000000000000000-0-0] [lt=16] start checking lost replicas [2024-12-02 13:55:17.481398] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC51-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:17.496069] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF603-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.499405] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BA-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:17.500502] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BA-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.500513] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BA-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:17.500527] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BA-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:17.500537] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BA-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:17.599796] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF604-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.605433] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BB-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:17.606386] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:345) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF604-0-0] [lt=53] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1020, ls_id:{id:1001}, sync_scn:{val:1732874426991810003, v:0}, readable_scn:{val:1732874426991810002, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:17.607634] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BB-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.607644] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BB-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:17.607657] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BB-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:17.607666] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BB-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:17.622719] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319550][LeaseQueueTh1][T0][YB42C0A800AF-000628430B4EEAC6-0-0] [lt=18] [DDL] execute ddl like stmt(ret=0, cost=16, ddl_arg=NULL) [2024-12-02 13:55:17.622772] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC31-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:17.625167] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC31-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:17.660766] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA86-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:17.663214] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA86-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:17.706873] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF605-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.712652] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BC-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:17.733958] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BC-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.733969] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BC-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:17.733986] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BC-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:17.733995] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BC-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:17.810849] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF606-0-0] [lt=21][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.838916] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BD-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:17.840457] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BD-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.840468] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BD-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:17.840483] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BD-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:17.840493] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BD-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:17.867987] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEACC-0-0] [lt=16] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:17.914554] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF607-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.945446] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BE-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:17.946645] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BE-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.946655] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BE-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:17.946670] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BE-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:17.946676] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BE-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:17.956532] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC43-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:18.017871] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF608-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.032273] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAC8-0-0] [lt=17] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:18.051621] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BF-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:18.052881] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC4B-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:18.053172] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BF-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.053183] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BF-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:18.053199] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BF-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:18.053209] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BF-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:18.136687] INFO [RS] get_dbms_sched_job_infos_in_tenant (ob_dbms_sched_table_operator.cpp:677) [2320057][T1_DBMSSched][T1][Y0-0000000000000000-0-0] [lt=19] failed to get result(ret=-4008) [2024-12-02 13:55:18.150849] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF609-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.158083] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C0-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:18.159252] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C0-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.159263] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C0-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:18.159278] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C0-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:18.159288] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C0-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:18.172807] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC31-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:18.175295] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC31-0-0] [lt=52] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:18.175314] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC31-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:18.195902] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAA0-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:18.212521] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB9-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:55:18.223632] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB9-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:55:18.236965] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC4F-0-0] [lt=35] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:18.254548] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF60A-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.264138] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C1-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:18.265394] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C1-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.265404] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C1-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:18.265419] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C1-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:18.265427] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C1-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:18.312925] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAA0-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:18.352731] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:18.352751] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=15] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=264, return_code_array=[0], broadcast_times=1) [2024-12-02 13:55:18.358425] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF60B-0-0] [lt=22][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.370290] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C2-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:18.371544] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C2-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.371554] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C2-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:18.371566] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C2-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:18.371571] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C2-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:18.462178] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF60C-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.476498] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C3-0-0] [lt=28] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:18.477703] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C3-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.477713] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C3-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:18.477728] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C3-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:18.477736] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C3-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:18.481476] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC52-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:18.566049] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF60D-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.566260] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF60D-0-0] [lt=46] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874426991810002, v:0}}], last_dump_ts_=1733118913566166, force_dump=false) [2024-12-02 13:55:18.582529] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C4-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:18.584504] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C4-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.584511] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C4-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:18.584525] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C4-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:18.584533] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C4-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:18.630487] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC32-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:18.632858] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC32-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:18.668804] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF60E-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.669834] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA87-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:18.672216] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA87-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:18.689239] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C5-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:18.690483] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C5-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.690490] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C5-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:18.690502] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C5-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:18.690512] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C5-0-0] [lt=10] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:18.772113] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF60F-0-0] [lt=11][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.795437] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C6-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:18.955559] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C6-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.955570] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C6-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:18.955582] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C6-0-0] [lt=11][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:18.955588] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C6-0-0] [lt=6] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:18.957942] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC44-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:19.016907] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF610-0-0] [lt=43] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874408973130000, v:0}}], last_dump_ts_=1733118913784625, force_dump=false) [2024-12-02 13:55:19.016944] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF610-0-0] [lt=11][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.052955] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC4C-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:19.060371] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C7-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:19.062539] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C7-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.062550] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C7-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:19.062567] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C7-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:19.062576] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C7-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:19.120968] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF611-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.167387] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C8-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:19.168640] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C8-0-0] [lt=13][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.168651] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C8-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:19.168668] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C8-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:19.168677] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C8-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:19.177914] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC32-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:19.180381] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC32-0-0] [lt=52] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:19.180399] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC32-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:19.195977] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAA1-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:19.224559] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF612-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.228836] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECBA-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:55:19.234643] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECBA-0-0] [lt=16] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118919068906001, v:0}}], last_dump_ts_=1733118914133892, force_dump=false) [2024-12-02 13:55:19.234667] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECBA-0-0] [lt=21] is primary tenant, no need check readable_scn [2024-12-02 13:55:19.237062] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC50-0-0] [lt=34] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:19.273502] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C9-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:19.274967] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C9-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.274974] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C9-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:19.274989] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C9-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:19.274995] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C9-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:19.291137] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320556][T1_FrzInfoDet][T1][YB42C0A800AF-00062843088EE998-0-0] [lt=17] get number(ret=0, service_type_=0, range=1, base_id=1733118919290940000, start_id=1733118919290940000, end_id=1733118919290940001) [2024-12-02 13:55:19.302397] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320556][T1_FrzInfoDet][T1][YB42C0A800AF-00062843088EE998-0-0] [lt=21] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118919290940000, v:0}, tenant_id=1) [2024-12-02 13:55:19.312999] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAA1-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:19.328124] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF613-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.352565] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118918067229001, replayable_scn:1733118918067229001, readable_scn:1733118918067229000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118919349867, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118918067229001, replayable_scn:1733118918067229001, readable_scn:1733118918067229000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118919349867, ora_rowscn_=1733118919231017000, content_changed=true) [2024-12-02 13:55:19.352831] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=20] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:19.379848] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CA-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:19.381088] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CA-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.381099] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CA-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:19.381115] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CA-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:19.381125] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CA-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:19.411460] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=18] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118919349867, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118918067229001, replayable_scn:1733118918067229001, readable_scn:1733118918067229000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:55:19.431665] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF614-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.481550] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC53-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:19.486067] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CB-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:19.487800] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CB-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.487807] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CB-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:19.487824] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CB-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:19.487833] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CB-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:19.535007] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF615-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.592713] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CC-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:19.601668] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CC-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.601678] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CC-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:19.601694] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CC-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:19.601703] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CC-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:19.623139] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319550][LeaseQueueTh1][T0][YB42C0A800AF-000628430B4EEAC7-0-0] [lt=19] [DDL] execute ddl like stmt(ret=0, cost=14, ddl_arg=NULL) [2024-12-02 13:55:19.638068] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC33-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:19.638714] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF616-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.640476] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC33-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:19.678929] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA88-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:19.681352] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA88-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:19.706440] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CD-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:19.708242] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CD-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.708252] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CD-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:19.708269] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CD-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:19.708278] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CD-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:19.742278] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF617-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.813101] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CE-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:19.814277] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CE-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.814288] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CE-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:19.814303] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CE-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:19.814308] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CE-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:19.846196] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF618-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.858272] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118919855057, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118919855057, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:55:19.872822] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEACD-0-0] [lt=23] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:19.919090] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CF-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:19.920807] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CF-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.920816] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CF-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:19.920832] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CF-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:19.920842] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CF-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:19.949849] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF619-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.959319] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC45-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:20.025666] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D0-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:20.027070] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D0-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.027081] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D0-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:20.027097] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D0-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:20.027106] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D0-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:20.032614] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAC9-0-0] [lt=16] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:20.053038] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC4D-0-0] [lt=21] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:20.053602] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF61A-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.131899] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D1-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:20.133400] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D1-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.133407] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D1-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:20.133418] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D1-0-0] [lt=9][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:20.133427] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D1-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:20.156972] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF61B-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.182997] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC33-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:20.185435] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC33-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:20.185454] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC33-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:20.196057] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAA2-0-0] [lt=21] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:20.237160] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC51-0-0] [lt=33] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:20.238223] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D2-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:20.239453] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D2-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.239463] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D2-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:20.239478] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D2-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:20.239484] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D2-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:20.239646] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECBB-0-0] [lt=9] is primary tenant, no need check readable_scn [2024-12-02 13:55:20.245168] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECBB-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:55:20.260901] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF61C-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.313076] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAA2-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:20.344310] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D3-0-0] [lt=14] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:20.345722] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D3-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.345733] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D3-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:20.345750] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D3-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:20.345759] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D3-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:20.352864] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:20.364729] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF61D-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.450499] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D4-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:20.451941] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D4-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.451952] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D4-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:20.451968] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D4-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:20.451977] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D4-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:20.468124] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF61E-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.481627] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC54-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:20.556747] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D5-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:20.558997] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D5-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.559007] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D5-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:20.559023] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D5-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:20.559033] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D5-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:20.571346] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF61F-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.645674] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC34-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:20.648070] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC34-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:20.663794] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D6-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:20.665296] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D6-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.665304] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D6-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:20.665319] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D6-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:20.665328] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D6-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:20.675078] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF620-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.688051] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA89-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:20.690479] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA89-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:20.770073] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D7-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:20.771868] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D7-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.771878] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D7-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:20.771894] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D7-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:20.771906] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D7-0-0] [lt=11] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:20.778412] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF621-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.876712] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D8-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:20.878231] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D8-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.878238] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D8-0-0] [lt=7][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:20.878254] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D8-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:20.878263] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D8-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:20.882059] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF622-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.960070] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1020, epoch_=0, all_service_names_=[]) [2024-12-02 13:55:20.960636] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC46-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:20.983067] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D9-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:20.984375] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D9-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.984386] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D9-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:20.984402] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D9-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:20.984410] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D9-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:20.985341] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF623-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.992528] INFO [RS] get_dbms_sched_job_infos_in_tenant (ob_dbms_sched_table_operator.cpp:677) [2324878][T1019_DBMSSched][T1019][Y0-0000000000000000-0-0] [lt=18] failed to get result(ret=-4008) [2024-12-02 13:55:21.053118] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC4E-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:21.088742] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF624-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.089363] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DA-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:21.090630] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DA-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.090640] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DA-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:21.090653] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DA-0-0] [lt=11][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:21.090663] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DA-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:21.132978] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=6] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:21.134385] WDIAG [RS] get_dbms_job_infos_in_tenant (ob_dbms_job_utils.cpp:363) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=6][errcode=-5019] execute query failed(ret=-5019, sql=select * from __all_job, tenant_id=1020) [2024-12-02 13:55:21.134443] WDIAG [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:613) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] fail to exec job_utils_.get_dbms_job_infos_in_tenant(tenant_id, allocator, job_infos)(ret=-5019) [2024-12-02 13:55:21.134455] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] load and register new jobs(ret=-5019, job_key=NULL, job_key=NULL, job_infos=[]) [2024-12-02 13:55:21.134466] WDIAG [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:590) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10][errcode=-5019] fail to exec load_and_register_new_jobs(tenant_ids.at(i))(ret=-5019) [2024-12-02 13:55:21.134470] INFO [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:593) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=4] register single tenant(ret=-5019, tenant_ids.at(i)=1020, tenant_schema->get_compatibility_mode()=0) [2024-12-02 13:55:21.134478] WDIAG [RS] scheduler_job (ob_dbms_job_master.cpp:368) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=5][errcode=-5019] fail to exec load_and_register_all_jobs(job_key)(ret=-5019) [2024-12-02 13:55:21.134485] WDIAG [RS] scheduler (ob_dbms_job_master.cpp:340) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=6][errcode=0] fail to scheduler single dbms job(ret=0, tmp_ret=-5019, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1733118926132998, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:55:21.188233] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC34-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:21.190804] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC34-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:21.190827] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC34-0-0] [lt=22] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:21.191502] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF625-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.195672] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DB-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:21.196133] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAA3-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:21.196234] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DB-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.196245] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DB-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:21.196262] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DB-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:21.196271] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DB-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:21.212762] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:21.213901] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118921212698, delay:5000000, job_id:0, tenant_id:1019}, job_key=0xfffeb4af6120, job_infos=[]) [2024-12-02 13:55:21.213925] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=20] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118926213922, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:55:21.221600] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2324931][T1019_FrzInfoDe][T1019][YB42C0A800AF-00062843139EE96E-0-0] [lt=17] get number(ret=0, service_type_=0, range=1, base_id=1733118921220846000, start_id=1733118921220846000, end_id=1733118921220846001) [2024-12-02 13:55:21.223143] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2324931][T1019_FrzInfoDe][T1019][YB42C0A800AF-00062843139EE96E-0-0] [lt=17] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118921220846000, v:0}, tenant_id=1019) [2024-12-02 13:55:21.237263] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC52-0-0] [lt=38] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:21.262487] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECBC-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:55:21.268397] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECBC-0-0] [lt=27] is primary tenant, no need check readable_scn [2024-12-02 13:55:21.295283] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF626-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.301295] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DC-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:21.302552] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DC-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.302563] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DC-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:21.302578] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DC-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:21.302587] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DC-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:21.307654] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:21.308790] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118921307590, delay:5000000, job_id:0, tenant_id:1001}, job_key=0xfffeb4af60a0, job_infos=[]) [2024-12-02 13:55:21.308815] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=23] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118926308813, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-02 13:55:21.309932] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118921308695, delay:5000000, job_id:0, tenant_id:1002}, job_key=0xfffeb4af60e0, job_infos=[]) [2024-12-02 13:55:21.309955] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=21] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118926309953, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-02 13:55:21.313154] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAA3-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:21.352779] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:21.354068] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-02 13:55:21.398571] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF627-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.407544] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DD-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:21.409071] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DD-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.409082] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DD-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:21.409098] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DD-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:21.409106] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DD-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:21.481700] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC55-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:21.502846] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF628-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.514111] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DE-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:21.515609] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DE-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.515620] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DE-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:21.515636] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DE-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:21.515645] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DE-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:21.596874] INFO [RS] get_dbms_sched_job_infos_in_tenant (ob_dbms_sched_table_operator.cpp:677) [2320250][T1001_DBMSSched][T1001][Y0-0000000000000000-0-0] [lt=21] failed to get result(ret=-4008) [2024-12-02 13:55:21.606152] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF629-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.620676] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DF-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:21.621846] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DF-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.621853] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DF-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:21.621869] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DF-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:21.621879] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DF-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:21.623556] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319550][LeaseQueueTh1][T0][YB42C0A800AF-000628430B4EEAC8-0-0] [lt=18] [DDL] execute ddl like stmt(ret=0, cost=13, ddl_arg=NULL) [2024-12-02 13:55:21.653425] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC35-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:21.655826] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC35-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:21.697187] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA8A-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:21.699588] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA8A-0-0] [lt=27] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:21.709919] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF62A-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.726751] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E0-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:21.728253] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E0-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.728264] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E0-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:21.728279] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E0-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:21.728288] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E0-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:21.813448] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF62B-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.833193] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E1-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:21.834522] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E1-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.834533] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E1-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:21.834548] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E1-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:21.834557] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E1-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:21.877484] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEACE-0-0] [lt=20] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:21.916794] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF62C-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.939571] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E2-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:21.941731] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E2-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.941741] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E2-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:21.941753] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E2-0-0] [lt=11][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:21.941761] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E2-0-0] [lt=6] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:21.961981] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC47-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:22.020234] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF62D-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.032933] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEACA-0-0] [lt=16] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:22.046622] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E3-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:22.047921] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E3-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.047928] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E3-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:22.047940] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E3-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:22.047945] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E3-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:22.053198] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC4F-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:22.123747] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF62E-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.152889] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E4-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:22.154612] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E4-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.154621] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E4-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:22.154636] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E4-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:22.154643] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E4-0-0] [lt=6] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:22.193503] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC35-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:22.195987] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC35-0-0] [lt=51] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:22.196003] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC35-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:22.196208] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAA4-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:22.227296] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF62F-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.237336] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC53-0-0] [lt=44] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:22.259699] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E5-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:22.269530] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E5-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.269537] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E5-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:22.269548] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E5-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:22.269553] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E5-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:22.289075] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECBD-0-0] [lt=11] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118922071217001, v:0}}], last_dump_ts_=1733118917180358, force_dump=false) [2024-12-02 13:55:22.289098] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECBD-0-0] [lt=18] is primary tenant, no need check readable_scn [2024-12-02 13:55:22.295886] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECBD-0-0] [lt=15] is primary tenant, no need check readable_scn [2024-12-02 13:55:22.313229] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAA4-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:22.330730] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF630-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.352696] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118921070407001, replayable_scn:1733118921070407001, readable_scn:1733118921070407001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118922349950, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118921070407001, replayable_scn:1733118921070407001, readable_scn:1733118921070407001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118922349950, ora_rowscn_=1733118922292078000, content_changed=true) [2024-12-02 13:55:22.352912] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=23] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:22.352929] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=208, return_code_array=[0], broadcast_times=2) [2024-12-02 13:55:22.374519] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E6-0-0] [lt=14] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:22.376135] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E6-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.376145] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E6-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:22.376160] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E6-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:22.376169] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E6-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:22.413103] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=17] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118922349950, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118921070407001, replayable_scn:1733118921070407001, readable_scn:1733118921070407001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:55:22.434040] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF631-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.468488] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320098][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDD9-0-0] [lt=17] [DDL] execute ddl like stmt(ret=0, cost=11, ddl_arg=NULL) [2024-12-02 13:55:22.481233] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E7-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:22.481776] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC56-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:22.482757] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E7-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.482767] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E7-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:22.482782] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E7-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:22.482790] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E7-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:22.537478] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF632-0-0] [lt=24][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.587683] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E8-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:22.589265] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E8-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.589274] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E8-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:22.589289] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E8-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:22.589295] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E8-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:22.640854] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF633-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.660933] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC36-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:22.663297] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC36-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:22.694237] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E9-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:22.695775] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E9-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.695786] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E9-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:22.695802] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E9-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:22.695811] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E9-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:22.706336] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA8B-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:22.708761] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA8B-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:22.744198] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF634-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.800687] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EA-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:22.801959] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EA-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.801970] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EA-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:22.801986] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EA-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:22.801995] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EA-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:22.847857] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF635-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.859929] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118922857298, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118922857298, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:55:22.906891] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EB-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:22.908380] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EB-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.908391] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EB-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:22.908406] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EB-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:22.908416] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EB-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:22.953028] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF636-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.963337] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC48-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:22.984308] INFO [SHARE] adaptive_update_limit (ob_share_throttle_define.cpp:112) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE998-0-0] [lt=20] adaptive update(Tenant ID=1001, Config Specify Resource Limit(MB)=511, TxShare Current Memory Limit(MB)=360, Holding Memory(MB)=61, Tenant Remain Memory(MB)=497, Usable Remain Memory(MB)=298, Last Update Limit Timestamp=1733118922983459, Is Updated=true) [2024-12-02 13:55:22.984471] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE998-0-0] [lt=21] get number(ret=0, service_type_=0, range=1, base_id=1733118922983459000, start_id=1733118922983459000, end_id=1733118922983459001) [2024-12-02 13:55:23.002281] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE998-0-0] [lt=17] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118922983459000, v:0}, tenant_id=1001) [2024-12-02 13:55:23.013392] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EC-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:23.014861] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EC-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.014872] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EC-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:23.014888] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EC-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:23.014897] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EC-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:23.053276] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC50-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:23.056874] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF637-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.119800] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2ED-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:23.121119] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2ED-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.121129] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2ED-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:23.121145] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2ED-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:23.121155] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2ED-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:23.160240] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF638-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.196285] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAA5-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:23.198673] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC36-0-0] [lt=23] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:23.201137] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC36-0-0] [lt=50] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:23.201156] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC36-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:23.226062] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EE-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:23.227709] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EE-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.227720] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EE-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:23.227736] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EE-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:23.227746] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EE-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:23.237437] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC54-0-0] [lt=38] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:23.263653] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF639-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.301219] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECBE-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:55:23.306471] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECBE-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:55:23.313304] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAA5-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:23.332657] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EF-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:23.333901] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EF-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.333911] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EF-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:23.333927] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EF-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:23.333936] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EF-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:23.352966] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:23.367005] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF63A-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.438912] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F0-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:23.440942] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F0-0-0] [lt=11][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.440953] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F0-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:23.440969] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F0-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:23.440979] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F0-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:23.470595] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF63B-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.481850] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC57-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:23.545902] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F1-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:23.547509] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F1-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.547520] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F1-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:23.547535] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F1-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:23.547544] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F1-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:23.574033] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF63C-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.574241] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF63C-0-0] [lt=48] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874426991810002, v:0}}], last_dump_ts_=1733118918566259, force_dump=false) [2024-12-02 13:55:23.623981] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319551][LeaseQueueTh2][T0][YB42C0A800AF-000628430B4EEAC9-0-0] [lt=19] [DDL] execute ddl like stmt(ret=0, cost=15, ddl_arg=NULL) [2024-12-02 13:55:23.632049] INFO [RS] get_dbms_sched_job_infos_in_tenant (ob_dbms_sched_table_operator.cpp:677) [2320439][T1002_DBMSSched][T1002][Y0-0000000000000000-0-0] [lt=22] failed to get result(ret=-4008) [2024-12-02 13:55:23.652430] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F2-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:23.653667] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F2-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.653678] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F2-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:23.653693] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F2-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:23.653699] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F2-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:23.668385] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC37-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:23.670803] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC37-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:23.678258] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF63D-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.715521] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA8C-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:23.719045] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA8C-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:23.758368] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F3-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:23.774539] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F3-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.774550] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F3-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:23.774565] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F3-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:23.774573] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F3-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:23.781606] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF63E-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.797039] INFO [RS.LB] idle (ob_root_balancer.cpp:103) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=23] root balance idle(idle_time=10000000) [2024-12-02 13:55:23.797063] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2074) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=15] start try disaster recovery [2024-12-02 13:55:23.797074] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=9] start try tenant disaster recovery(tenant_id=1, only_for_display=false) [2024-12-02 13:55:23.798307] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=8] start try ls disaster recovery(ls_status_info={tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:23.802864] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=23] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:23.802892] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=22] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.802902] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:1733118159456434, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:23.802956] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=49] build ls locality stat map(tenant_id=1, ls_id={id:1}) [2024-12-02 13:55:23.802966] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:23.802978] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118159456434, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:23.803013] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.803028] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.803037] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.803045] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.804962] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=7] try tenant disaster recovery succ(tenant_id=1, only_for_display=false) [2024-12-02 13:55:23.804980] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=17] start try tenant disaster recovery(tenant_id=1001, only_for_display=false) [2024-12-02 13:55:23.806258] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1001, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:23.810682] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=23] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1001, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:23.810709] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=21] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.810720] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1001, ls_id:{id:1}, replicas:[{modify_time_us:1733118160462607, create_time_us:1732871267341550, tenant_id:1001, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1001, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:23.810772] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=49] build ls locality stat map(tenant_id=1001, ls_id={id:1}) [2024-12-02 13:55:23.810785] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:23.810797] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=10] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118160462607, create_time_us:1732871267341550, tenant_id:1001, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:23.810831] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.810846] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.810855] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.810862] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.812723] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=7] try tenant disaster recovery succ(tenant_id=1001, only_for_display=false) [2024-12-02 13:55:23.814023] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=13] start try tenant disaster recovery(tenant_id=1002, only_for_display=false) [2024-12-02 13:55:23.815227] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=18] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:23.819597] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=22] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:23.819625] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=22] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.819636] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1}, replicas:[{modify_time_us:1733118167177128, create_time_us:1732871280733688, tenant_id:1002, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:23.819689] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=50] build ls locality stat map(tenant_id=1002, ls_id={id:1}) [2024-12-02 13:55:23.819701] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:23.819713] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118167177128, create_time_us:1732871280733688, tenant_id:1002, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:23.819746] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=25] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.819761] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.819769] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.819776] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.821648] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:23.828328] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=22] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1001}, acc_dr_task=0) [2024-12-02 13:55:23.828356] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=23] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.828367] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1001}, replicas:[{modify_time_us:1733118167179939, create_time_us:1732871299864461, tenant_id:1002, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:23.828420] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=50] build ls locality stat map(tenant_id=1002, ls_id={id:1001}) [2024-12-02 13:55:23.828429] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:23.828442] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=10] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118167179939, create_time_us:1732871299864461, tenant_id:1002, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:23.828474] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=26] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.828489] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.828497] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.828505] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12